vector_core/config/
proxy.rs

1use headers::{Authorization, authorization::Credentials};
2use http::{header::PROXY_AUTHORIZATION, uri::InvalidUri};
3use hyper_proxy::{Custom, Intercept, Proxy, ProxyConnector};
4use no_proxy::NoProxy;
5use url::Url;
6use vector_config::configurable_component;
7
8use crate::serde::is_default;
9
10// suggestion of standardization coming from https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/
11fn from_env(key: &str) -> Option<String> {
12    // use lowercase first and the uppercase
13    std::env::var(key.to_lowercase())
14        .ok()
15        .or_else(|| std::env::var(key.to_uppercase()).ok())
16}
17
18#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug, PartialEq, Eq)]
19pub struct NoProxyInterceptor(NoProxy);
20
21impl NoProxyInterceptor {
22    fn intercept(self, expected_scheme: &'static str) -> Intercept {
23        Intercept::Custom(Custom::from(
24            move |scheme: Option<&str>, host: Option<&str>, port: Option<u16>| {
25                if scheme.is_some() && scheme != Some(expected_scheme) {
26                    return false;
27                }
28                let matches = host.is_some_and(|host| {
29                    self.0.matches(host)
30                        || port.is_some_and(|port| {
31                            let url = format!("{host}:{port}");
32                            self.0.matches(&url)
33                        })
34                });
35                // only intercept those that don't match
36                !matches
37            },
38        ))
39    }
40}
41
42/// Proxy configuration.
43///
44/// Configure to proxy traffic through an HTTP(S) proxy when making external requests.
45///
46/// Similar to common proxy configuration convention, you can set different proxies
47/// to use based on the type of traffic being proxied. You can also set specific hosts that
48/// should not be proxied.
49#[configurable_component]
50#[derive(Clone, Debug, Eq, PartialEq)]
51#[serde(deny_unknown_fields)]
52pub struct ProxyConfig {
53    /// Enables proxying support.
54    #[serde(
55        default = "ProxyConfig::default_enabled",
56        skip_serializing_if = "is_enabled"
57    )]
58    pub enabled: bool,
59
60    /// Proxy endpoint to use when proxying HTTP traffic.
61    ///
62    /// Must be a valid URI string.
63    #[configurable(validation(format = "uri"))]
64    #[configurable(metadata(docs::examples = "http://foo.bar:3128"))]
65    #[serde(default, skip_serializing_if = "is_default")]
66    pub http: Option<String>,
67
68    /// Proxy endpoint to use when proxying HTTPS traffic.
69    ///
70    /// Must be a valid URI string.
71    #[configurable(validation(format = "uri"))]
72    #[serde(default, skip_serializing_if = "is_default")]
73    #[configurable(metadata(docs::examples = "http://foo.bar:3128"))]
74    pub https: Option<String>,
75
76    /// A list of hosts to avoid proxying.
77    ///
78    /// Multiple patterns are allowed:
79    ///
80    /// | Pattern             | Example match                                                               |
81    /// | ------------------- | --------------------------------------------------------------------------- |
82    /// | Domain names        | `example.com` matches requests to `example.com`                     |
83    /// | Wildcard domains    | `.example.com` matches requests to `example.com` and its subdomains |
84    /// | IP addresses        | `127.0.0.1` matches requests to `127.0.0.1`                         |
85    /// | [CIDR][cidr] blocks | `192.168.0.0/16` matches requests to any IP addresses in this range     |
86    /// | Splat               | `*` matches all hosts                                                   |
87    ///
88    /// [cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing
89    #[serde(default, skip_serializing_if = "is_default")]
90    #[configurable(metadata(docs::examples = "localhost"))]
91    #[configurable(metadata(docs::examples = ".foo.bar"))]
92    #[configurable(metadata(docs::examples = "*"))]
93    pub no_proxy: NoProxy,
94}
95
96impl Default for ProxyConfig {
97    fn default() -> Self {
98        Self {
99            enabled: Self::default_enabled(),
100            http: None,
101            https: None,
102            no_proxy: NoProxy::default(),
103        }
104    }
105}
106
107#[allow(clippy::trivially_copy_pass_by_ref)] // Calling convention is required by serde
108fn is_enabled(e: &bool) -> bool {
109    e == &true
110}
111
112impl ProxyConfig {
113    fn default_enabled() -> bool {
114        true
115    }
116
117    pub fn from_env() -> Self {
118        Self {
119            enabled: true,
120            http: from_env("HTTP_PROXY"),
121            https: from_env("HTTPS_PROXY"),
122            no_proxy: from_env("NO_PROXY").map(NoProxy::from).unwrap_or_default(),
123        }
124    }
125
126    pub fn merge_with_env(global: &Self, component: &Self) -> Self {
127        Self::from_env().merge(&global.merge(component))
128    }
129
130    fn interceptor(&self) -> NoProxyInterceptor {
131        NoProxyInterceptor(self.no_proxy.clone())
132    }
133
134    // overrides current proxy configuration with other configuration
135    // if `self` is the global config and `other` the component config,
136    // if both have the `http` proxy set, the one from `other` should be kept
137    #[must_use]
138    pub fn merge(&self, other: &Self) -> Self {
139        let no_proxy = if other.no_proxy.is_empty() {
140            self.no_proxy.clone()
141        } else {
142            other.no_proxy.clone()
143        };
144
145        Self {
146            enabled: self.enabled && other.enabled,
147            http: other.http.clone().or_else(|| self.http.clone()),
148            https: other.https.clone().or_else(|| self.https.clone()),
149            no_proxy,
150        }
151    }
152
153    fn build_proxy(
154        &self,
155        proxy_scheme: &'static str,
156        proxy_url: Option<&String>,
157    ) -> Result<Option<Proxy>, InvalidUri> {
158        proxy_url
159            .as_ref()
160            .map(|url| {
161                url.parse().map(|parsed| {
162                    let mut proxy = Proxy::new(self.interceptor().intercept(proxy_scheme), parsed);
163                    if let Ok(authority) = Url::parse(url)
164                        && let Some(password) = authority.password()
165                    {
166                        let decoded_user = urlencoding::decode(authority.username())
167                            .expect("username must be valid UTF-8.");
168                        let decoded_pw =
169                            urlencoding::decode(password).expect("Password must be valid UTF-8.");
170                        let mut authorization =
171                            Authorization::basic(&decoded_user, &decoded_pw).0.encode();
172                        authorization.set_sensitive(true);
173                        proxy.set_header(PROXY_AUTHORIZATION, authorization);
174                    }
175                    proxy
176                })
177            })
178            .transpose()
179    }
180
181    fn http_proxy(&self) -> Result<Option<Proxy>, InvalidUri> {
182        self.build_proxy("http", self.http.as_ref())
183    }
184
185    fn https_proxy(&self) -> Result<Option<Proxy>, InvalidUri> {
186        self.build_proxy("https", self.https.as_ref())
187    }
188
189    /// Install the [`ProxyConnector<C>`] for this `ProxyConfig`
190    ///
191    /// # Errors
192    ///
193    /// Function will error if passed `ProxyConnector` has a faulty URI.
194    pub fn configure<C>(&self, connector: &mut ProxyConnector<C>) -> Result<(), InvalidUri> {
195        if self.enabled {
196            if let Some(proxy) = self.http_proxy()? {
197                connector.add_proxy(proxy);
198            }
199            if let Some(proxy) = self.https_proxy()? {
200                connector.add_proxy(proxy);
201            }
202        }
203        Ok(())
204    }
205}
206
207#[cfg(test)]
208mod tests {
209    use base64::prelude::{BASE64_STANDARD, Engine as _};
210    use env_test_util::TempEnvVar;
211    use http::{HeaderValue, Uri, header::AUTHORIZATION};
212    use proptest::prelude::*;
213
214    use super::*;
215
216    impl Arbitrary for ProxyConfig {
217        type Parameters = ();
218        type Strategy = BoxedStrategy<Self>;
219
220        fn arbitrary_with((): Self::Parameters) -> Self::Strategy {
221            (
222                any::<bool>(),
223                any::<Option<String>>(),
224                any::<Option<String>>(),
225            )
226                .prop_map(|(enabled, http, https)| Self {
227                    enabled,
228                    http,
229                    https,
230                    // TODO: Neither NoProxy nor IpCidr contained with in it supports proptest. Once
231                    // they support proptest, add another any here.
232                    no_proxy: Default::default(),
233                })
234                .boxed()
235        }
236    }
237
238    proptest! {
239        #[test]
240        fn encodes_and_decodes_through_yaml(config:ProxyConfig) {
241            let yaml = serde_yaml::to_string(&config).expect("Could not serialize config");
242            let reloaded: ProxyConfig = serde_yaml::from_str(&yaml)
243                .expect("Could not deserialize config");
244            assert_eq!(config, reloaded);
245        }
246    }
247
248    #[test]
249    fn merge_simple() {
250        let first = ProxyConfig::default();
251        let second = ProxyConfig {
252            https: Some("https://2.3.4.5:9876".into()),
253            ..Default::default()
254        };
255        let result = first.merge(&second);
256        assert_eq!(result.http, None);
257        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
258    }
259
260    #[test]
261    fn merge_fill() {
262        // coming from env
263        let first = ProxyConfig {
264            http: Some("http://1.2.3.4:5678".into()),
265            ..Default::default()
266        };
267        // global config
268        let second = ProxyConfig {
269            https: Some("https://2.3.4.5:9876".into()),
270            ..Default::default()
271        };
272        // component config
273        let third = ProxyConfig {
274            no_proxy: NoProxy::from("localhost"),
275            ..Default::default()
276        };
277        let result = first.merge(&second).merge(&third);
278        assert_eq!(result.http, Some("http://1.2.3.4:5678".into()));
279        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
280        assert!(result.no_proxy.matches("localhost"));
281    }
282
283    #[test]
284    fn merge_override() {
285        let first = ProxyConfig {
286            http: Some("http://1.2.3.4:5678".into()),
287            no_proxy: NoProxy::from("127.0.0.1,google.com"),
288            ..Default::default()
289        };
290        let second = ProxyConfig {
291            http: Some("http://1.2.3.4:5678".into()),
292            https: Some("https://2.3.4.5:9876".into()),
293            no_proxy: NoProxy::from("localhost"),
294            ..Default::default()
295        };
296        let result = first.merge(&second);
297        assert_eq!(result.http, Some("http://1.2.3.4:5678".into()));
298        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
299        assert!(!result.no_proxy.matches("127.0.0.1"));
300        assert!(result.no_proxy.matches("localhost"));
301    }
302
303    #[test]
304    fn with_environment_variables() {
305        let global_proxy = ProxyConfig {
306            http: Some("http://1.2.3.4:5678".into()),
307            ..Default::default()
308        };
309        let component_proxy = ProxyConfig {
310            https: Some("https://2.3.4.5:9876".into()),
311            ..Default::default()
312        };
313        let _http = TempEnvVar::new("HTTP_PROXY").with("http://remote.proxy");
314        let _https = TempEnvVar::new("HTTPS_PROXY");
315        let result = ProxyConfig::merge_with_env(&global_proxy, &component_proxy);
316
317        assert_eq!(result.http, Some("http://1.2.3.4:5678".into()));
318        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
319
320        // with the component proxy disabled
321        let global_proxy = ProxyConfig {
322            https: Some("https://2.3.4.5:9876".into()),
323            ..Default::default()
324        };
325        let component_proxy = ProxyConfig {
326            enabled: false,
327            ..Default::default()
328        };
329        let result = ProxyConfig::merge_with_env(&global_proxy, &component_proxy);
330
331        assert!(!result.enabled);
332        assert_eq!(result.http, Some("http://remote.proxy".into()));
333        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
334    }
335
336    #[test]
337    fn build_proxy() {
338        let config = ProxyConfig {
339            http: Some("http://1.2.3.4:5678".into()),
340            https: Some("https://2.3.4.5:9876".into()),
341            ..Default::default()
342        };
343        let first = config
344            .http_proxy()
345            .expect("should not be an error")
346            .expect("should not be None");
347        let second = config
348            .https_proxy()
349            .expect("should not be an error")
350            .expect("should not be None");
351
352        assert_eq!(
353            Some(first.uri()),
354            Uri::try_from("http://1.2.3.4:5678").as_ref().ok()
355        );
356        assert_eq!(
357            Some(second.uri()),
358            Uri::try_from("https://2.3.4.5:9876").as_ref().ok()
359        );
360    }
361
362    #[test]
363    fn build_proxy_with_basic_authorization() {
364        let config = ProxyConfig {
365            http: Some("http://user:pass@1.2.3.4:5678".into()),
366            https: Some("https://user:pass@2.3.4.5:9876".into()),
367            ..Default::default()
368        };
369        let first = config
370            .http_proxy()
371            .expect("should not be an error")
372            .expect("should not be None");
373        let second = config
374            .https_proxy()
375            .expect("should not be an error")
376            .expect("should not be None");
377        let encoded_header = format!("Basic {}", BASE64_STANDARD.encode("user:pass"));
378        let expected_header_value = HeaderValue::from_str(encoded_header.as_str());
379
380        assert_eq!(
381            Some(first.uri()),
382            Uri::try_from("http://user:pass@1.2.3.4:5678").as_ref().ok()
383        );
384        assert_eq!(
385            first.headers().get(PROXY_AUTHORIZATION),
386            expected_header_value.as_ref().ok()
387        );
388        assert!(!first.headers().contains_key(AUTHORIZATION));
389        assert_eq!(
390            Some(second.uri()),
391            Uri::try_from("https://user:pass@2.3.4.5:9876")
392                .as_ref()
393                .ok()
394        );
395        assert_eq!(
396            second.headers().get(PROXY_AUTHORIZATION),
397            expected_header_value.as_ref().ok()
398        );
399        assert!(!second.headers().contains_key(AUTHORIZATION));
400    }
401
402    #[test]
403    fn build_proxy_with_special_chars_url_encoded() {
404        let config = ProxyConfig {
405            http: Some("http://user:P%40ssw0rd@1.2.3.4:5678".into()),
406            https: Some("https://user:P%40ssw0rd@2.3.4.5:9876".into()),
407            ..Default::default()
408        };
409        let first = config
410            .http_proxy()
411            .expect("should not be an error")
412            .expect("should not be None");
413        let encoded_header = format!("Basic {}", BASE64_STANDARD.encode("user:P@ssw0rd"));
414        let expected_header_value = HeaderValue::from_str(encoded_header.as_str());
415        assert_eq!(
416            first.headers().get(PROXY_AUTHORIZATION),
417            expected_header_value.as_ref().ok()
418        );
419        assert!(!first.headers().contains_key(AUTHORIZATION));
420    }
421}
vector_core/config/proxy.rs

vector_core/config/
proxy.rs
