vector_core/config/
proxy.rs

1use headers::Authorization;
2use http::uri::InvalidUri;
3use hyper_proxy::{Custom, Intercept, Proxy, ProxyConnector};
4use no_proxy::NoProxy;
5use url::Url;
6use vector_config::configurable_component;
7
8use crate::serde::is_default;
9
10// suggestion of standardization coming from https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/
11fn from_env(key: &str) -> Option<String> {
12    // use lowercase first and the uppercase
13    std::env::var(key.to_lowercase())
14        .ok()
15        .or_else(|| std::env::var(key.to_uppercase()).ok())
16}
17
18#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug, PartialEq, Eq)]
19pub struct NoProxyInterceptor(NoProxy);
20
21impl NoProxyInterceptor {
22    fn intercept(self, expected_scheme: &'static str) -> Intercept {
23        Intercept::Custom(Custom::from(
24            move |scheme: Option<&str>, host: Option<&str>, port: Option<u16>| {
25                if scheme.is_some() && scheme != Some(expected_scheme) {
26                    return false;
27                }
28                let matches = host.is_some_and(|host| {
29                    self.0.matches(host)
30                        || port.is_some_and(|port| {
31                            let url = format!("{host}:{port}");
32                            self.0.matches(&url)
33                        })
34                });
35                // only intercept those that don't match
36                !matches
37            },
38        ))
39    }
40}
41
42/// Proxy configuration.
43///
44/// Configure to proxy traffic through an HTTP(S) proxy when making external requests.
45///
46/// Similar to common proxy configuration convention, you can set different proxies
47/// to use based on the type of traffic being proxied. You can also set specific hosts that
48/// should not be proxied.
49#[configurable_component]
50#[configurable(metadata(docs::advanced))]
51#[derive(Clone, Debug, Eq, PartialEq)]
52#[serde(deny_unknown_fields)]
53pub struct ProxyConfig {
54    /// Enables proxying support.
55    #[serde(
56        default = "ProxyConfig::default_enabled",
57        skip_serializing_if = "is_enabled"
58    )]
59    pub enabled: bool,
60
61    /// Proxy endpoint to use when proxying HTTP traffic.
62    ///
63    /// Must be a valid URI string.
64    #[configurable(validation(format = "uri"))]
65    #[configurable(metadata(docs::examples = "http://foo.bar:3128"))]
66    #[serde(default, skip_serializing_if = "is_default")]
67    pub http: Option<String>,
68
69    /// Proxy endpoint to use when proxying HTTPS traffic.
70    ///
71    /// Must be a valid URI string.
72    #[configurable(validation(format = "uri"))]
73    #[serde(default, skip_serializing_if = "is_default")]
74    #[configurable(metadata(docs::examples = "http://foo.bar:3128"))]
75    pub https: Option<String>,
76
77    /// A list of hosts to avoid proxying.
78    ///
79    /// Multiple patterns are allowed:
80    ///
81    /// | Pattern             | Example match                                                               |
82    /// | ------------------- | --------------------------------------------------------------------------- |
83    /// | Domain names        | `example.com` matches requests to `example.com`                     |
84    /// | Wildcard domains    | `.example.com` matches requests to `example.com` and its subdomains |
85    /// | IP addresses        | `127.0.0.1` matches requests to `127.0.0.1`                         |
86    /// | [CIDR][cidr] blocks | `192.168.0.0/16` matches requests to any IP addresses in this range     |
87    /// | Splat               | `*` matches all hosts                                                   |
88    ///
89    /// [cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing
90    #[serde(default, skip_serializing_if = "is_default")]
91    #[configurable(metadata(docs::examples = "localhost"))]
92    #[configurable(metadata(docs::examples = ".foo.bar"))]
93    #[configurable(metadata(docs::examples = "*"))]
94    pub no_proxy: NoProxy,
95}
96
97impl Default for ProxyConfig {
98    fn default() -> Self {
99        Self {
100            enabled: Self::default_enabled(),
101            http: None,
102            https: None,
103            no_proxy: NoProxy::default(),
104        }
105    }
106}
107
108#[allow(clippy::trivially_copy_pass_by_ref)] // Calling convention is required by serde
109fn is_enabled(e: &bool) -> bool {
110    e == &true
111}
112
113impl ProxyConfig {
114    fn default_enabled() -> bool {
115        true
116    }
117
118    pub fn from_env() -> Self {
119        Self {
120            enabled: true,
121            http: from_env("HTTP_PROXY"),
122            https: from_env("HTTPS_PROXY"),
123            no_proxy: from_env("NO_PROXY").map(NoProxy::from).unwrap_or_default(),
124        }
125    }
126
127    pub fn merge_with_env(global: &Self, component: &Self) -> Self {
128        Self::from_env().merge(&global.merge(component))
129    }
130
131    fn interceptor(&self) -> NoProxyInterceptor {
132        NoProxyInterceptor(self.no_proxy.clone())
133    }
134
135    // overrides current proxy configuration with other configuration
136    // if `self` is the global config and `other` the component config,
137    // if both have the `http` proxy set, the one from `other` should be kept
138    #[must_use]
139    pub fn merge(&self, other: &Self) -> Self {
140        let no_proxy = if other.no_proxy.is_empty() {
141            self.no_proxy.clone()
142        } else {
143            other.no_proxy.clone()
144        };
145
146        Self {
147            enabled: self.enabled && other.enabled,
148            http: other.http.clone().or_else(|| self.http.clone()),
149            https: other.https.clone().or_else(|| self.https.clone()),
150            no_proxy,
151        }
152    }
153
154    fn build_proxy(
155        &self,
156        proxy_scheme: &'static str,
157        proxy_url: Option<&String>,
158    ) -> Result<Option<Proxy>, InvalidUri> {
159        proxy_url
160            .as_ref()
161            .map(|url| {
162                url.parse().map(|parsed| {
163                    let mut proxy = Proxy::new(self.interceptor().intercept(proxy_scheme), parsed);
164                    if let Ok(authority) = Url::parse(url) {
165                        if let Some(password) = authority.password() {
166                            let decoded_user = urlencoding::decode(authority.username())
167                                .expect("username must be valid UTF-8.");
168                            let decoded_pw = urlencoding::decode(password)
169                                .expect("Password must be valid UTF-8.");
170                            proxy.set_authorization(Authorization::basic(
171                                &decoded_user,
172                                &decoded_pw,
173                            ));
174                        }
175                    }
176                    proxy
177                })
178            })
179            .transpose()
180    }
181
182    fn http_proxy(&self) -> Result<Option<Proxy>, InvalidUri> {
183        self.build_proxy("http", self.http.as_ref())
184    }
185
186    fn https_proxy(&self) -> Result<Option<Proxy>, InvalidUri> {
187        self.build_proxy("https", self.https.as_ref())
188    }
189
190    /// Install the [`ProxyConnector<C>`] for this `ProxyConfig`
191    ///
192    /// # Errors
193    ///
194    /// Function will error if passed `ProxyConnector` has a faulty URI.
195    pub fn configure<C>(&self, connector: &mut ProxyConnector<C>) -> Result<(), InvalidUri> {
196        if self.enabled {
197            if let Some(proxy) = self.http_proxy()? {
198                connector.add_proxy(proxy);
199            }
200            if let Some(proxy) = self.https_proxy()? {
201                connector.add_proxy(proxy);
202            }
203        }
204        Ok(())
205    }
206}
207
208#[cfg(test)]
209mod tests {
210    use base64::prelude::{Engine as _, BASE64_STANDARD};
211    use env_test_util::TempEnvVar;
212    use http::{
213        header::{AUTHORIZATION, PROXY_AUTHORIZATION},
214        HeaderName, HeaderValue, Uri,
215    };
216    use proptest::prelude::*;
217
218    const PROXY_HEADERS: [HeaderName; 2] = [AUTHORIZATION, PROXY_AUTHORIZATION];
219
220    use super::*;
221
222    impl Arbitrary for ProxyConfig {
223        type Parameters = ();
224        type Strategy = BoxedStrategy<Self>;
225
226        fn arbitrary_with((): Self::Parameters) -> Self::Strategy {
227            (
228                any::<bool>(),
229                any::<Option<String>>(),
230                any::<Option<String>>(),
231            )
232                .prop_map(|(enabled, http, https)| Self {
233                    enabled,
234                    http,
235                    https,
236                    // TODO: Neither NoProxy nor IpCidr contained with in it supports proptest. Once
237                    // they support proptest, add another any here.
238                    no_proxy: Default::default(),
239                })
240                .boxed()
241        }
242    }
243
244    proptest! {
245        #[test]
246        fn encodes_and_decodes_through_yaml(config:ProxyConfig) {
247            let yaml = serde_yaml::to_string(&config).expect("Could not serialize config");
248            let reloaded: ProxyConfig = serde_yaml::from_str(&yaml)
249                .expect("Could not deserialize config");
250            assert_eq!(config, reloaded);
251        }
252    }
253
254    #[test]
255    fn merge_simple() {
256        let first = ProxyConfig::default();
257        let second = ProxyConfig {
258            https: Some("https://2.3.4.5:9876".into()),
259            ..Default::default()
260        };
261        let result = first.merge(&second);
262        assert_eq!(result.http, None);
263        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
264    }
265
266    #[test]
267    fn merge_fill() {
268        // coming from env
269        let first = ProxyConfig {
270            http: Some("http://1.2.3.4:5678".into()),
271            ..Default::default()
272        };
273        // global config
274        let second = ProxyConfig {
275            https: Some("https://2.3.4.5:9876".into()),
276            ..Default::default()
277        };
278        // component config
279        let third = ProxyConfig {
280            no_proxy: NoProxy::from("localhost"),
281            ..Default::default()
282        };
283        let result = first.merge(&second).merge(&third);
284        assert_eq!(result.http, Some("http://1.2.3.4:5678".into()));
285        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
286        assert!(result.no_proxy.matches("localhost"));
287    }
288
289    #[test]
290    fn merge_override() {
291        let first = ProxyConfig {
292            http: Some("http://1.2.3.4:5678".into()),
293            no_proxy: NoProxy::from("127.0.0.1,google.com"),
294            ..Default::default()
295        };
296        let second = ProxyConfig {
297            http: Some("http://1.2.3.4:5678".into()),
298            https: Some("https://2.3.4.5:9876".into()),
299            no_proxy: NoProxy::from("localhost"),
300            ..Default::default()
301        };
302        let result = first.merge(&second);
303        assert_eq!(result.http, Some("http://1.2.3.4:5678".into()));
304        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
305        assert!(!result.no_proxy.matches("127.0.0.1"));
306        assert!(result.no_proxy.matches("localhost"));
307    }
308
309    #[test]
310    fn with_environment_variables() {
311        let global_proxy = ProxyConfig {
312            http: Some("http://1.2.3.4:5678".into()),
313            ..Default::default()
314        };
315        let component_proxy = ProxyConfig {
316            https: Some("https://2.3.4.5:9876".into()),
317            ..Default::default()
318        };
319        let _http = TempEnvVar::new("HTTP_PROXY").with("http://remote.proxy");
320        let _https = TempEnvVar::new("HTTPS_PROXY");
321        let result = ProxyConfig::merge_with_env(&global_proxy, &component_proxy);
322
323        assert_eq!(result.http, Some("http://1.2.3.4:5678".into()));
324        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
325
326        // with the component proxy disabled
327        let global_proxy = ProxyConfig {
328            https: Some("https://2.3.4.5:9876".into()),
329            ..Default::default()
330        };
331        let component_proxy = ProxyConfig {
332            enabled: false,
333            ..Default::default()
334        };
335        let result = ProxyConfig::merge_with_env(&global_proxy, &component_proxy);
336
337        assert!(!result.enabled);
338        assert_eq!(result.http, Some("http://remote.proxy".into()));
339        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
340    }
341
342    #[test]
343    fn build_proxy() {
344        let config = ProxyConfig {
345            http: Some("http://1.2.3.4:5678".into()),
346            https: Some("https://2.3.4.5:9876".into()),
347            ..Default::default()
348        };
349        let first = config
350            .http_proxy()
351            .expect("should not be an error")
352            .expect("should not be None");
353        let second = config
354            .https_proxy()
355            .expect("should not be an error")
356            .expect("should not be None");
357
358        assert_eq!(
359            Some(first.uri()),
360            Uri::try_from("http://1.2.3.4:5678").as_ref().ok()
361        );
362        assert_eq!(
363            Some(second.uri()),
364            Uri::try_from("https://2.3.4.5:9876").as_ref().ok()
365        );
366    }
367
368    #[test]
369    fn build_proxy_with_basic_authorization() {
370        let config = ProxyConfig {
371            http: Some("http://user:pass@1.2.3.4:5678".into()),
372            https: Some("https://user:pass@2.3.4.5:9876".into()),
373            ..Default::default()
374        };
375        let first = config
376            .http_proxy()
377            .expect("should not be an error")
378            .expect("should not be None");
379        let second = config
380            .https_proxy()
381            .expect("should not be an error")
382            .expect("should not be None");
383        let encoded_header = format!("Basic {}", BASE64_STANDARD.encode("user:pass"));
384        let expected_header_value = HeaderValue::from_str(encoded_header.as_str());
385
386        assert_eq!(
387            Some(first.uri()),
388            Uri::try_from("http://user:pass@1.2.3.4:5678").as_ref().ok()
389        );
390        for h in &PROXY_HEADERS {
391            assert_eq!(first.headers().get(h), expected_header_value.as_ref().ok());
392        }
393        assert_eq!(
394            Some(second.uri()),
395            Uri::try_from("https://user:pass@2.3.4.5:9876")
396                .as_ref()
397                .ok()
398        );
399        for h in &PROXY_HEADERS {
400            assert_eq!(second.headers().get(h), expected_header_value.as_ref().ok());
401        }
402    }
403
404    #[test]
405    fn build_proxy_with_special_chars_url_encoded() {
406        let config = ProxyConfig {
407            http: Some("http://user:P%40ssw0rd@1.2.3.4:5678".into()),
408            https: Some("https://user:P%40ssw0rd@2.3.4.5:9876".into()),
409            ..Default::default()
410        };
411        let first = config
412            .http_proxy()
413            .expect("should not be an error")
414            .expect("should not be None");
415        let encoded_header = format!("Basic {}", BASE64_STANDARD.encode("user:P@ssw0rd"));
416        let expected_header_value = HeaderValue::from_str(encoded_header.as_str());
417        for h in &PROXY_HEADERS {
418            assert_eq!(first.headers().get(h), expected_header_value.as_ref().ok());
419        }
420    }
421}
vector_core/config/proxy.rs

vector_core/config/
proxy.rs
