vector_core/config/
proxy.rs

1use headers::Authorization;
2use http::uri::InvalidUri;
3use hyper_proxy::{Custom, Intercept, Proxy, ProxyConnector};
4use no_proxy::NoProxy;
5use url::Url;
6use vector_config::configurable_component;
7
8use crate::serde::is_default;
9
10// suggestion of standardization coming from https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/
11fn from_env(key: &str) -> Option<String> {
12    // use lowercase first and the uppercase
13    std::env::var(key.to_lowercase())
14        .ok()
15        .or_else(|| std::env::var(key.to_uppercase()).ok())
16}
17
18#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug, PartialEq, Eq)]
19pub struct NoProxyInterceptor(NoProxy);
20
21impl NoProxyInterceptor {
22    fn intercept(self, expected_scheme: &'static str) -> Intercept {
23        Intercept::Custom(Custom::from(
24            move |scheme: Option<&str>, host: Option<&str>, port: Option<u16>| {
25                if scheme.is_some() && scheme != Some(expected_scheme) {
26                    return false;
27                }
28                let matches = host.is_some_and(|host| {
29                    self.0.matches(host)
30                        || port.is_some_and(|port| {
31                            let url = format!("{host}:{port}");
32                            self.0.matches(&url)
33                        })
34                });
35                // only intercept those that don't match
36                !matches
37            },
38        ))
39    }
40}
41
42/// Proxy configuration.
43///
44/// Configure to proxy traffic through an HTTP(S) proxy when making external requests.
45///
46/// Similar to common proxy configuration convention, you can set different proxies
47/// to use based on the type of traffic being proxied. You can also set specific hosts that
48/// should not be proxied.
49#[configurable_component]
50#[configurable(metadata(docs::advanced))]
51#[derive(Clone, Debug, Eq, PartialEq)]
52#[serde(deny_unknown_fields)]
53pub struct ProxyConfig {
54    /// Enables proxying support.
55    #[serde(
56        default = "ProxyConfig::default_enabled",
57        skip_serializing_if = "is_enabled"
58    )]
59    pub enabled: bool,
60
61    /// Proxy endpoint to use when proxying HTTP traffic.
62    ///
63    /// Must be a valid URI string.
64    #[configurable(validation(format = "uri"))]
65    #[configurable(metadata(docs::examples = "http://foo.bar:3128"))]
66    #[serde(default, skip_serializing_if = "is_default")]
67    pub http: Option<String>,
68
69    /// Proxy endpoint to use when proxying HTTPS traffic.
70    ///
71    /// Must be a valid URI string.
72    #[configurable(validation(format = "uri"))]
73    #[serde(default, skip_serializing_if = "is_default")]
74    #[configurable(metadata(docs::examples = "http://foo.bar:3128"))]
75    pub https: Option<String>,
76
77    /// A list of hosts to avoid proxying.
78    ///
79    /// Multiple patterns are allowed:
80    ///
81    /// | Pattern             | Example match                                                               |
82    /// | ------------------- | --------------------------------------------------------------------------- |
83    /// | Domain names        | `example.com` matches requests to `example.com`                     |
84    /// | Wildcard domains    | `.example.com` matches requests to `example.com` and its subdomains |
85    /// | IP addresses        | `127.0.0.1` matches requests to `127.0.0.1`                         |
86    /// | [CIDR][cidr] blocks | `192.168.0.0/16` matches requests to any IP addresses in this range     |
87    /// | Splat               | `*` matches all hosts                                                   |
88    ///
89    /// [cidr]: https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing
90    #[serde(default, skip_serializing_if = "is_default")]
91    #[configurable(metadata(docs::examples = "localhost"))]
92    #[configurable(metadata(docs::examples = ".foo.bar"))]
93    #[configurable(metadata(docs::examples = "*"))]
94    pub no_proxy: NoProxy,
95}
96
97impl Default for ProxyConfig {
98    fn default() -> Self {
99        Self {
100            enabled: Self::default_enabled(),
101            http: None,
102            https: None,
103            no_proxy: NoProxy::default(),
104        }
105    }
106}
107
108#[allow(clippy::trivially_copy_pass_by_ref)] // Calling convention is required by serde
109fn is_enabled(e: &bool) -> bool {
110    e == &true
111}
112
113impl ProxyConfig {
114    fn default_enabled() -> bool {
115        true
116    }
117
118    pub fn from_env() -> Self {
119        Self {
120            enabled: true,
121            http: from_env("HTTP_PROXY"),
122            https: from_env("HTTPS_PROXY"),
123            no_proxy: from_env("NO_PROXY").map(NoProxy::from).unwrap_or_default(),
124        }
125    }
126
127    pub fn merge_with_env(global: &Self, component: &Self) -> Self {
128        Self::from_env().merge(&global.merge(component))
129    }
130
131    fn interceptor(&self) -> NoProxyInterceptor {
132        NoProxyInterceptor(self.no_proxy.clone())
133    }
134
135    // overrides current proxy configuration with other configuration
136    // if `self` is the global config and `other` the component config,
137    // if both have the `http` proxy set, the one from `other` should be kept
138    #[must_use]
139    pub fn merge(&self, other: &Self) -> Self {
140        let no_proxy = if other.no_proxy.is_empty() {
141            self.no_proxy.clone()
142        } else {
143            other.no_proxy.clone()
144        };
145
146        Self {
147            enabled: self.enabled && other.enabled,
148            http: other.http.clone().or_else(|| self.http.clone()),
149            https: other.https.clone().or_else(|| self.https.clone()),
150            no_proxy,
151        }
152    }
153
154    fn build_proxy(
155        &self,
156        proxy_scheme: &'static str,
157        proxy_url: Option<&String>,
158    ) -> Result<Option<Proxy>, InvalidUri> {
159        proxy_url
160            .as_ref()
161            .map(|url| {
162                url.parse().map(|parsed| {
163                    let mut proxy = Proxy::new(self.interceptor().intercept(proxy_scheme), parsed);
164                    if let Ok(authority) = Url::parse(url)
165                        && let Some(password) = authority.password()
166                    {
167                        let decoded_user = urlencoding::decode(authority.username())
168                            .expect("username must be valid UTF-8.");
169                        let decoded_pw =
170                            urlencoding::decode(password).expect("Password must be valid UTF-8.");
171                        proxy.set_authorization(Authorization::basic(&decoded_user, &decoded_pw));
172                    }
173                    proxy
174                })
175            })
176            .transpose()
177    }
178
179    fn http_proxy(&self) -> Result<Option<Proxy>, InvalidUri> {
180        self.build_proxy("http", self.http.as_ref())
181    }
182
183    fn https_proxy(&self) -> Result<Option<Proxy>, InvalidUri> {
184        self.build_proxy("https", self.https.as_ref())
185    }
186
187    /// Install the [`ProxyConnector<C>`] for this `ProxyConfig`
188    ///
189    /// # Errors
190    ///
191    /// Function will error if passed `ProxyConnector` has a faulty URI.
192    pub fn configure<C>(&self, connector: &mut ProxyConnector<C>) -> Result<(), InvalidUri> {
193        if self.enabled {
194            if let Some(proxy) = self.http_proxy()? {
195                connector.add_proxy(proxy);
196            }
197            if let Some(proxy) = self.https_proxy()? {
198                connector.add_proxy(proxy);
199            }
200        }
201        Ok(())
202    }
203}
204
205#[cfg(test)]
206mod tests {
207    use base64::prelude::{BASE64_STANDARD, Engine as _};
208    use env_test_util::TempEnvVar;
209    use http::{
210        HeaderName, HeaderValue, Uri,
211        header::{AUTHORIZATION, PROXY_AUTHORIZATION},
212    };
213    use proptest::prelude::*;
214
215    const PROXY_HEADERS: [HeaderName; 2] = [AUTHORIZATION, PROXY_AUTHORIZATION];
216
217    use super::*;
218
219    impl Arbitrary for ProxyConfig {
220        type Parameters = ();
221        type Strategy = BoxedStrategy<Self>;
222
223        fn arbitrary_with((): Self::Parameters) -> Self::Strategy {
224            (
225                any::<bool>(),
226                any::<Option<String>>(),
227                any::<Option<String>>(),
228            )
229                .prop_map(|(enabled, http, https)| Self {
230                    enabled,
231                    http,
232                    https,
233                    // TODO: Neither NoProxy nor IpCidr contained with in it supports proptest. Once
234                    // they support proptest, add another any here.
235                    no_proxy: Default::default(),
236                })
237                .boxed()
238        }
239    }
240
241    proptest! {
242        #[test]
243        fn encodes_and_decodes_through_yaml(config:ProxyConfig) {
244            let yaml = serde_yaml::to_string(&config).expect("Could not serialize config");
245            let reloaded: ProxyConfig = serde_yaml::from_str(&yaml)
246                .expect("Could not deserialize config");
247            assert_eq!(config, reloaded);
248        }
249    }
250
251    #[test]
252    fn merge_simple() {
253        let first = ProxyConfig::default();
254        let second = ProxyConfig {
255            https: Some("https://2.3.4.5:9876".into()),
256            ..Default::default()
257        };
258        let result = first.merge(&second);
259        assert_eq!(result.http, None);
260        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
261    }
262
263    #[test]
264    fn merge_fill() {
265        // coming from env
266        let first = ProxyConfig {
267            http: Some("http://1.2.3.4:5678".into()),
268            ..Default::default()
269        };
270        // global config
271        let second = ProxyConfig {
272            https: Some("https://2.3.4.5:9876".into()),
273            ..Default::default()
274        };
275        // component config
276        let third = ProxyConfig {
277            no_proxy: NoProxy::from("localhost"),
278            ..Default::default()
279        };
280        let result = first.merge(&second).merge(&third);
281        assert_eq!(result.http, Some("http://1.2.3.4:5678".into()));
282        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
283        assert!(result.no_proxy.matches("localhost"));
284    }
285
286    #[test]
287    fn merge_override() {
288        let first = ProxyConfig {
289            http: Some("http://1.2.3.4:5678".into()),
290            no_proxy: NoProxy::from("127.0.0.1,google.com"),
291            ..Default::default()
292        };
293        let second = ProxyConfig {
294            http: Some("http://1.2.3.4:5678".into()),
295            https: Some("https://2.3.4.5:9876".into()),
296            no_proxy: NoProxy::from("localhost"),
297            ..Default::default()
298        };
299        let result = first.merge(&second);
300        assert_eq!(result.http, Some("http://1.2.3.4:5678".into()));
301        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
302        assert!(!result.no_proxy.matches("127.0.0.1"));
303        assert!(result.no_proxy.matches("localhost"));
304    }
305
306    #[test]
307    fn with_environment_variables() {
308        let global_proxy = ProxyConfig {
309            http: Some("http://1.2.3.4:5678".into()),
310            ..Default::default()
311        };
312        let component_proxy = ProxyConfig {
313            https: Some("https://2.3.4.5:9876".into()),
314            ..Default::default()
315        };
316        let _http = TempEnvVar::new("HTTP_PROXY").with("http://remote.proxy");
317        let _https = TempEnvVar::new("HTTPS_PROXY");
318        let result = ProxyConfig::merge_with_env(&global_proxy, &component_proxy);
319
320        assert_eq!(result.http, Some("http://1.2.3.4:5678".into()));
321        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
322
323        // with the component proxy disabled
324        let global_proxy = ProxyConfig {
325            https: Some("https://2.3.4.5:9876".into()),
326            ..Default::default()
327        };
328        let component_proxy = ProxyConfig {
329            enabled: false,
330            ..Default::default()
331        };
332        let result = ProxyConfig::merge_with_env(&global_proxy, &component_proxy);
333
334        assert!(!result.enabled);
335        assert_eq!(result.http, Some("http://remote.proxy".into()));
336        assert_eq!(result.https, Some("https://2.3.4.5:9876".into()));
337    }
338
339    #[test]
340    fn build_proxy() {
341        let config = ProxyConfig {
342            http: Some("http://1.2.3.4:5678".into()),
343            https: Some("https://2.3.4.5:9876".into()),
344            ..Default::default()
345        };
346        let first = config
347            .http_proxy()
348            .expect("should not be an error")
349            .expect("should not be None");
350        let second = config
351            .https_proxy()
352            .expect("should not be an error")
353            .expect("should not be None");
354
355        assert_eq!(
356            Some(first.uri()),
357            Uri::try_from("http://1.2.3.4:5678").as_ref().ok()
358        );
359        assert_eq!(
360            Some(second.uri()),
361            Uri::try_from("https://2.3.4.5:9876").as_ref().ok()
362        );
363    }
364
365    #[test]
366    fn build_proxy_with_basic_authorization() {
367        let config = ProxyConfig {
368            http: Some("http://user:pass@1.2.3.4:5678".into()),
369            https: Some("https://user:pass@2.3.4.5:9876".into()),
370            ..Default::default()
371        };
372        let first = config
373            .http_proxy()
374            .expect("should not be an error")
375            .expect("should not be None");
376        let second = config
377            .https_proxy()
378            .expect("should not be an error")
379            .expect("should not be None");
380        let encoded_header = format!("Basic {}", BASE64_STANDARD.encode("user:pass"));
381        let expected_header_value = HeaderValue::from_str(encoded_header.as_str());
382
383        assert_eq!(
384            Some(first.uri()),
385            Uri::try_from("http://user:pass@1.2.3.4:5678").as_ref().ok()
386        );
387        for h in &PROXY_HEADERS {
388            assert_eq!(first.headers().get(h), expected_header_value.as_ref().ok());
389        }
390        assert_eq!(
391            Some(second.uri()),
392            Uri::try_from("https://user:pass@2.3.4.5:9876")
393                .as_ref()
394                .ok()
395        );
396        for h in &PROXY_HEADERS {
397            assert_eq!(second.headers().get(h), expected_header_value.as_ref().ok());
398        }
399    }
400
401    #[test]
402    fn build_proxy_with_special_chars_url_encoded() {
403        let config = ProxyConfig {
404            http: Some("http://user:P%40ssw0rd@1.2.3.4:5678".into()),
405            https: Some("https://user:P%40ssw0rd@2.3.4.5:9876".into()),
406            ..Default::default()
407        };
408        let first = config
409            .http_proxy()
410            .expect("should not be an error")
411            .expect("should not be None");
412        let encoded_header = format!("Basic {}", BASE64_STANDARD.encode("user:P@ssw0rd"));
413        let expected_header_value = HeaderValue::from_str(encoded_header.as_str());
414        for h in &PROXY_HEADERS {
415            assert_eq!(first.headers().get(h), expected_header_value.as_ref().ok());
416        }
417    }
418}
vector_core/config/proxy.rs

vector_core/config/
proxy.rs
