vector/config/loading/
secret.rs

1use std::{
2    collections::{HashMap, HashSet},
3    io::Read,
4    sync::LazyLock,
5};
6
7use futures::TryFutureExt;
8use indexmap::IndexMap;
9use regex::{Captures, Regex};
10use serde::{Deserialize, Serialize};
11use toml::value::Table;
12use vector_lib::config::ComponentKey;
13
14use crate::{
15    config::{
16        loading::{deserialize_table, prepare_input, process::Process, ComponentHint, Loader},
17        SecretBackend,
18    },
19    secrets::SecretBackends,
20    signal,
21};
22
23// The following regex aims to extract a pair of strings, the first being the secret backend name
24// and the second being the secret key. Here are some matching & non-matching examples:
25// - "SECRET[backend.secret_name]" will match and capture "backend" and "secret_name"
26// - "SECRET[backend.secret.name]" will match and capture "backend" and "secret.name"
27// - "SECRET[backend..secret.name]" will match and capture "backend" and ".secret.name"
28// - "SECRET[secret_name]" will not match
29// - "SECRET[.secret.name]" will not match
30pub static COLLECTOR: LazyLock<Regex> =
31    LazyLock::new(|| Regex::new(r"SECRET\[([[:word:]]+)\.([[:word:].-]+)\]").unwrap());
32
33/// Helper type for specifically deserializing secrets backends.
34#[derive(Debug, Default, Deserialize, Serialize)]
35pub(crate) struct SecretBackendOuter {
36    #[serde(default)]
37    pub(crate) secret: IndexMap<ComponentKey, SecretBackends>,
38}
39
40/// Loader for secrets backends.
41#[derive(Debug, Default, Deserialize, Serialize)]
42pub struct SecretBackendLoader {
43    backends: IndexMap<ComponentKey, SecretBackends>,
44    pub(crate) secret_keys: HashMap<String, HashSet<String>>,
45}
46
47impl SecretBackendLoader {
48    pub(crate) fn new() -> Self {
49        Self {
50            backends: IndexMap::new(),
51            secret_keys: HashMap::new(),
52        }
53    }
54
55    pub(crate) async fn retrieve(
56        &mut self,
57        signal_rx: &mut signal::SignalRx,
58    ) -> Result<HashMap<String, String>, String> {
59        let mut secrets: HashMap<String, String> = HashMap::new();
60
61        for (backend_name, keys) in &self.secret_keys {
62            let backend = self.backends
63                .get_mut(&ComponentKey::from(backend_name.clone()))
64                .ok_or_else(|| {
65                    format!("Backend \"{backend_name}\" is required for secret retrieval but was not found in config.")
66                })?;
67
68            debug!(message = "Retrieving secrets from a backend.", backend = ?backend_name, keys = ?keys);
69            let backend_secrets = backend
70                .retrieve(keys.clone(), signal_rx)
71                .map_err(|e| {
72                    format!("Error while retrieving secret from backend \"{backend_name}\": {e}.",)
73                })
74                .await?;
75
76            for (k, v) in backend_secrets {
77                trace!(message = "Successfully retrieved a secret.", backend = ?backend_name, key = ?k);
78                secrets.insert(format!("{backend_name}.{k}"), v);
79            }
80        }
81
82        Ok(secrets)
83    }
84
85    pub(crate) fn has_secrets_to_retrieve(&self) -> bool {
86        !self.secret_keys.is_empty()
87    }
88}
89
90impl Process for SecretBackendLoader {
91    fn prepare<R: Read>(&mut self, input: R) -> Result<String, Vec<String>> {
92        let config_string = prepare_input(input)?;
93        // Collect secret placeholders just after env var processing
94        collect_secret_keys(&config_string, &mut self.secret_keys);
95        Ok(config_string)
96    }
97
98    fn merge(&mut self, table: Table, _: Option<ComponentHint>) -> Result<(), Vec<String>> {
99        if table.contains_key("secret") {
100            let additional = deserialize_table::<SecretBackendOuter>(table)?;
101            self.backends.extend(additional.secret);
102        }
103        Ok(())
104    }
105}
106
107impl Loader<SecretBackendLoader> for SecretBackendLoader {
108    fn take(self) -> SecretBackendLoader {
109        self
110    }
111}
112
113fn collect_secret_keys(input: &str, keys: &mut HashMap<String, HashSet<String>>) {
114    COLLECTOR.captures_iter(input).for_each(|cap| {
115        if let (Some(backend), Some(key)) = (cap.get(1), cap.get(2)) {
116            if let Some(keys) = keys.get_mut(backend.as_str()) {
117                keys.insert(key.as_str().to_string());
118            } else {
119                keys.insert(
120                    backend.as_str().to_string(),
121                    HashSet::from_iter(std::iter::once(key.as_str().to_string())),
122                );
123            }
124        }
125    });
126}
127
128pub fn interpolate(input: &str, secrets: &HashMap<String, String>) -> Result<String, Vec<String>> {
129    let mut errors = Vec::<String>::new();
130    let output = COLLECTOR
131        .replace_all(input, |caps: &Captures<'_>| {
132            caps.get(1)
133                .and_then(|b| caps.get(2).map(|k| (b, k)))
134                .and_then(|(b, k)| secrets.get(&format!("{}.{}", b.as_str(), k.as_str())))
135                .cloned()
136                .unwrap_or_else(|| {
137                    errors.push(format!(
138                        "Unable to find secret replacement for {}.",
139                        caps.get(0).unwrap().as_str()
140                    ));
141                    "".to_string()
142                })
143        })
144        .into_owned();
145    if errors.is_empty() {
146        Ok(output)
147    } else {
148        Err(errors)
149    }
150}
151
152#[cfg(test)]
153mod tests {
154    use std::collections::HashMap;
155
156    use indoc::indoc;
157
158    use super::{collect_secret_keys, interpolate};
159
160    #[test]
161    fn replacement() {
162        let secrets: HashMap<String, String> = vec![
163            ("a.secret.key".into(), "value".into()),
164            ("a...key".into(), "a...value".into()),
165        ]
166        .into_iter()
167        .collect();
168
169        assert_eq!(
170            Ok("value".into()),
171            interpolate("SECRET[a.secret.key]", &secrets)
172        );
173        assert_eq!(
174            Ok("value value".into()),
175            interpolate("SECRET[a.secret.key] SECRET[a.secret.key]", &secrets)
176        );
177
178        assert_eq!(
179            Ok("xxxvalueyyy".into()),
180            interpolate("xxxSECRET[a.secret.key]yyy", &secrets)
181        );
182        assert_eq!(
183            Ok("a...value".into()),
184            interpolate("SECRET[a...key]", &secrets)
185        );
186        assert_eq!(
187            Ok("xxxSECRET[non_matching_syntax]yyy".into()),
188            interpolate("xxxSECRET[non_matching_syntax]yyy", &secrets)
189        );
190        assert_eq!(
191            Err(vec![
192                "Unable to find secret replacement for SECRET[a.non.existing.key].".into()
193            ]),
194            interpolate("xxxSECRET[a.non.existing.key]yyy", &secrets)
195        );
196    }
197
198    #[test]
199    fn collection() {
200        let mut keys = HashMap::new();
201        collect_secret_keys(
202            indoc! {r"
203            SECRET[first_backend.secret_key]
204            SECRET[first_backend.secret-key]
205            SECRET[first_backend.another_secret_key]
206            SECRET[second_backend.secret_key]
207            SECRET[second_backend.secret.key]
208            SECRET[first_backend.a_third.secret_key]
209            SECRET[first_backend...an_extra_secret_key]
210            SECRET[non_matching_syntax]
211            SECRET[.non.matching.syntax]
212        "},
213            &mut keys,
214        );
215        assert_eq!(keys.len(), 2);
216        assert!(keys.contains_key("first_backend"));
217        assert!(keys.contains_key("second_backend"));
218
219        let first_backend_keys = keys.get("first_backend").unwrap();
220        assert_eq!(first_backend_keys.len(), 5);
221        assert!(first_backend_keys.contains("secret_key"));
222        assert!(first_backend_keys.contains("secret-key"));
223        assert!(first_backend_keys.contains("another_secret_key"));
224        assert!(first_backend_keys.contains("a_third.secret_key"));
225        assert!(first_backend_keys.contains("..an_extra_secret_key"));
226
227        let second_backend_keys = keys.get("second_backend").unwrap();
228        assert_eq!(second_backend_keys.len(), 2);
229        assert!(second_backend_keys.contains("secret_key"));
230        assert!(second_backend_keys.contains("secret.key"));
231    }
232
233    #[test]
234    fn collection_duplicates() {
235        let mut keys = HashMap::new();
236        collect_secret_keys(
237            indoc! {r"
238            SECRET[first_backend.secret_key]
239            SECRET[first_backend.secret_key]
240        "},
241            &mut keys,
242        );
243
244        let first_backend_keys = keys.get("first_backend").unwrap();
245        assert_eq!(first_backend_keys.len(), 1);
246        assert!(first_backend_keys.contains("secret_key"));
247    }
248}
vector/config/loading/secret.rs

vector/config/loading/
secret.rs
