1use crate::compiler::prelude::*;
2use crate::core::encode_key_value;
3use crate::value::KeyString;
4use std::sync::LazyLock;
5
6pub(crate) fn encode_key_value(
8 fields: Option<Value>,
9 value: Value,
10 key_value_delimiter: &Value,
11 field_delimiter: &Value,
12 flatten_boolean: Value,
13) -> ExpressionResult<Value> {
14 let fields = match fields {
15 None => Ok(vec![]),
16 Some(fields) => resolve_fields(fields),
17 }?;
18 let object = value.try_object()?;
19 let key_value_delimiter = key_value_delimiter.try_bytes_utf8_lossy()?;
20 let field_delimiter = field_delimiter.try_bytes_utf8_lossy()?;
21 let flatten_boolean = flatten_boolean.try_boolean()?;
22 Ok(encode_key_value::to_string(
23 &object,
24 &fields[..],
25 &key_value_delimiter,
26 &field_delimiter,
27 flatten_boolean,
28 )
29 .expect("Should always succeed.")
30 .into())
31}
32
33pub(super) static DEFAULT_FIELDS_ORDERING: LazyLock<Value> = LazyLock::new(|| Value::Array(vec![]));
34static DEFAULT_KEY_VALUE_DELIMITER: LazyLock<Value> =
35 LazyLock::new(|| Value::Bytes(Bytes::from("=")));
36static DEFAULT_FIELD_DELIMITER: LazyLock<Value> = LazyLock::new(|| Value::Bytes(Bytes::from(" ")));
37static DEFAULT_FLATTEN_BOOLEAN: LazyLock<Value> = LazyLock::new(|| Value::Boolean(false));
38
39static PARAMETERS: LazyLock<Vec<Parameter>> = LazyLock::new(|| {
40 vec![
41 Parameter::required("value", kind::OBJECT, "The value to convert to a string."),
42 Parameter::optional("fields_ordering", kind::ARRAY, "The ordering of fields to preserve. Any fields not in this list are listed unordered, after all ordered fields.")
43 .default(&DEFAULT_FIELDS_ORDERING),
44 Parameter::optional("key_value_delimiter", kind::BYTES, "The string that separates the key from the value.")
45 .default(&DEFAULT_KEY_VALUE_DELIMITER),
46 Parameter::optional("field_delimiter", kind::BYTES, "The string that separates each key-value pair.")
47 .default(&DEFAULT_FIELD_DELIMITER),
48 Parameter::optional("flatten_boolean", kind::BOOLEAN, "Whether to encode key-value with a boolean value as a standalone key if `true` and nothing if `false`.")
49 .default(&DEFAULT_FLATTEN_BOOLEAN),
50 ]
51});
52
53#[derive(Clone, Copy, Debug)]
54pub struct EncodeKeyValue;
55
56impl Function for EncodeKeyValue {
57 fn identifier(&self) -> &'static str {
58 "encode_key_value"
59 }
60
61 fn usage(&self) -> &'static str {
62 "Encodes the `value` into key-value format with customizable delimiters. Default delimiters match the [logfmt](https://brandur.org/logfmt) format."
63 }
64
65 fn category(&self) -> &'static str {
66 Category::Codec.as_ref()
67 }
68
69 fn internal_failure_reasons(&self) -> &'static [&'static str] {
70 &["`fields_ordering` contains a non-string element."]
71 }
72
73 fn return_kind(&self) -> u16 {
74 kind::BYTES
75 }
76
77 fn notices(&self) -> &'static [&'static str] {
78 &["If `fields_ordering` is specified then the function is fallible else it is infallible."]
79 }
80
81 fn parameters(&self) -> &'static [Parameter] {
82 PARAMETERS.as_slice()
83 }
84
85 fn compile(
86 &self,
87 _state: &state::TypeState,
88 _ctx: &mut FunctionCompileContext,
89 arguments: ArgumentList,
90 ) -> Compiled {
91 let value = arguments.required("value");
92 let fields = arguments.optional("fields_ordering");
93
94 let key_value_delimiter = arguments.optional("key_value_delimiter");
95 let field_delimiter = arguments.optional("field_delimiter");
96 let flatten_boolean = arguments.optional("flatten_boolean");
97
98 Ok(EncodeKeyValueFn {
99 value,
100 fields,
101 key_value_delimiter,
102 field_delimiter,
103 flatten_boolean,
104 }
105 .as_expr())
106 }
107
108 fn examples(&self) -> &'static [Example] {
109 &[
110 example! {
111 title: "Encode with default delimiters (no ordering)",
112 source: indoc! {r#"
113 encode_key_value(
114 {
115 "ts": "2021-06-05T17:20:00Z",
116 "msg": "This is a message",
117 "lvl": "info"
118 }
119 )
120 "#},
121 result: Ok(r#"lvl=info msg="This is a message" ts=2021-06-05T17:20:00Z"#),
122 },
123 example! {
124 title: "Encode with default delimiters (fields ordering)",
125 source: indoc! {r#"
126 encode_key_value!(
127 {
128 "ts": "2021-06-05T17:20:00Z",
129 "msg": "This is a message",
130 "lvl": "info",
131 "log_id": 12345
132 },
133 ["ts", "lvl", "msg"]
134 )
135 "#},
136 result: Ok(r#"ts=2021-06-05T17:20:00Z lvl=info msg="This is a message" log_id=12345"#),
137 },
138 example! {
139 title: "Encode with default delimiters (nested fields)",
140 source: indoc! {r#"
141 encode_key_value(
142 {
143 "agent": {"name": "foo"},
144 "log": {"file": {"path": "my.log"}},
145 "event": "log"
146 }
147 )
148 "#},
149 result: Ok(r"agent.name=foo event=log log.file.path=my.log"),
150 },
151 example! {
152 title: "Encode with default delimiters (nested fields ordering)",
153 source: indoc! {r#"
154 encode_key_value!(
155 {
156 "agent": {"name": "foo"},
157 "log": {"file": {"path": "my.log"}},
158 "event": "log"
159 },
160 ["event", "log.file.path", "agent.name"])
161 "#},
162 result: Ok(r"event=log log.file.path=my.log agent.name=foo"),
163 },
164 example! {
165 title: "Encode with custom delimiters (no ordering)",
166 source: indoc! {r#"
167 encode_key_value(
168 {"ts": "2021-06-05T17:20:00Z", "msg": "This is a message", "lvl": "info"},
169 field_delimiter: ",",
170 key_value_delimiter: ":"
171 )
172 "#},
173 result: Ok(r#"lvl:info,msg:"This is a message",ts:2021-06-05T17:20:00Z"#),
174 },
175 example! {
176 title: "Encode with custom delimiters and flatten boolean",
177 source: indoc! {r#"
178 encode_key_value(
179 {"ts": "2021-06-05T17:20:00Z", "msg": "This is a message", "lvl": "info", "beta": true, "dropped": false},
180 field_delimiter: ",",
181 key_value_delimiter: ":",
182 flatten_boolean: true
183 )
184 "#},
185 result: Ok(r#"beta,lvl:info,msg:"This is a message",ts:2021-06-05T17:20:00Z"#),
186 },
187 ]
188 }
189}
190
191#[derive(Clone, Debug)]
192pub(crate) struct EncodeKeyValueFn {
193 pub(crate) value: Box<dyn Expression>,
194 pub(crate) fields: Option<Box<dyn Expression>>,
195 pub(crate) key_value_delimiter: Option<Box<dyn Expression>>,
196 pub(crate) field_delimiter: Option<Box<dyn Expression>>,
197 pub(crate) flatten_boolean: Option<Box<dyn Expression>>,
198}
199
200fn resolve_fields(fields: Value) -> ExpressionResult<Vec<KeyString>> {
201 let arr = fields.try_array()?;
202 arr.iter()
203 .enumerate()
204 .map(|(idx, v)| {
205 v.try_bytes_utf8_lossy()
206 .map(|v| v.to_string().into())
207 .map_err(|e| format!("invalid field value type at index {idx}: {e}").into())
208 })
209 .collect()
210}
211
212impl FunctionExpression for EncodeKeyValueFn {
213 fn resolve(&self, ctx: &mut Context) -> Resolved {
214 let value = self.value.resolve(ctx)?;
215 let fields = self
216 .fields
217 .map_resolve_with_default(ctx, || DEFAULT_FIELDS_ORDERING.clone())?;
218 let key_value_delimiter = self
219 .key_value_delimiter
220 .map_resolve_with_default(ctx, || DEFAULT_KEY_VALUE_DELIMITER.clone())?;
221 let field_delimiter = self
222 .field_delimiter
223 .map_resolve_with_default(ctx, || DEFAULT_FIELD_DELIMITER.clone())?;
224 let flatten_boolean = self
225 .flatten_boolean
226 .map_resolve_with_default(ctx, || DEFAULT_FLATTEN_BOOLEAN.clone())?;
227
228 encode_key_value(
229 Some(fields),
230 value,
231 &key_value_delimiter,
232 &field_delimiter,
233 flatten_boolean,
234 )
235 }
236
237 fn type_def(&self, _: &state::TypeState) -> TypeDef {
238 TypeDef::bytes().maybe_fallible(self.fields.is_some())
239 }
240}
241
242#[cfg(test)]
243mod tests {
244 use std::collections::BTreeMap;
245
246 use crate::{
247 btreemap,
248 stdlib::parse_key_value::{Whitespace, parse_key_value},
249 value,
250 };
251
252 use super::*;
253
254 #[test]
255 fn test_encode_decode_cycle() {
256 let before: Value = {
257 let mut map = Value::from(BTreeMap::default());
258 map.insert("key", r#"this has a " quote"#);
259 map
260 };
261
262 let after = parse_key_value(
263 &encode_key_value(None, before.clone(), &"=".into(), &" ".into(), true.into())
264 .expect("valid key value before"),
265 &Value::from("="),
266 &Value::from(" "),
267 true.into(),
268 Whitespace::Lenient,
269 )
270 .expect("valid key value after");
271
272 assert_eq!(before, after);
273 }
274
275 #[test]
276 fn test_decode_encode_cycle() {
277 let before: Value = r#"key="this has a \" quote""#.into();
278
279 let after = encode_key_value(
280 Some(Value::Array(vec![
281 "key".into(),
282 "has".into(),
283 "a".into(),
284 r#"""#.into(),
285 "quote".into(),
286 ])),
287 parse_key_value(
288 &before,
289 &Value::from("="),
290 &Value::from(" "),
291 true.into(),
292 Whitespace::Lenient,
293 )
294 .expect("valid key value before"),
295 &Value::from("="),
296 &Value::from(" "),
297 true.into(),
298 )
299 .expect("valid key value after");
300
301 assert_eq!(before, after);
302 }
303
304 test_function![
305 encode_key_value => EncodeKeyValue;
306
307 single_element {
308 args: func_args![value:
309 btreemap! {
310 "lvl" => "info"
311 }
312 ],
313 want: Ok("lvl=info"),
314 tdef: TypeDef::bytes().infallible(),
315 }
316
317 multiple_elements {
318 args: func_args![value:
319 btreemap! {
320 "lvl" => "info",
321 "log_id" => 12345
322 }
323 ],
324 want: Ok("log_id=12345 lvl=info"),
325 tdef: TypeDef::bytes().infallible(),
326 }
327
328 string_with_spaces {
329 args: func_args![value:
330 btreemap! {
331 "lvl" => "info",
332 "msg" => "This is a log message"
333 }],
334 want: Ok(r#"lvl=info msg="This is a log message""#),
335 tdef: TypeDef::bytes().infallible(),
336 }
337
338 string_with_quotes {
339 args: func_args![value:
340 btreemap! {
341 "lvl" => "info",
342 "msg" => "{\"key\":\"value\"}"
343 }],
344 want: Ok(r#"lvl=info msg="{\"key\":\"value\"}""#),
345 tdef: TypeDef::bytes().infallible(),
346 }
347
348 flatten_boolean {
349 args: func_args![value:
350 btreemap! {
351 "beta" => true,
352 "prod" => false,
353 "lvl" => "info",
354 "msg" => "This is a log message",
355 },
356 flatten_boolean: value!(true)
357 ],
358 want: Ok(r#"beta lvl=info msg="This is a log message""#),
359 tdef: TypeDef::bytes().infallible(),
360 }
361
362 dont_flatten_boolean {
363 args: func_args![value:
364 btreemap! {
365 "beta" => true,
366 "prod" => false,
367 "lvl" => "info",
368 "msg" => "This is a log message",
369 },
370 flatten_boolean: value!(false)
371 ],
372 want: Ok(r#"beta=true lvl=info msg="This is a log message" prod=false"#),
373 tdef: TypeDef::bytes().infallible(),
374 }
375
376 flatten_boolean_with_custom_delimiters {
377 args: func_args![value:
378 btreemap! {
379 "tag_a" => "val_a",
380 "tag_b" => "val_b",
381 "tag_c" => true,
382 },
383 key_value_delimiter: value!(":"),
384 field_delimiter: value!(","),
385 flatten_boolean: value!(true)
386 ],
387 want: Ok("tag_a:val_a,tag_b:val_b,tag_c"),
388 tdef: TypeDef::bytes().infallible(),
389 }
390 string_with_characters_to_escape {
391 args: func_args![value:
392 btreemap! {
393 "lvl" => "info",
394 "msg" => r#"payload: {"code": 200}\n"#,
395 "another_field" => "some\nfield\\and things",
396 "space key" => "foo"
397 }],
398 want: Ok(r#"another_field="some\\nfield\\and things" lvl=info msg="payload: {\"code\": 200}\\n" "space key"=foo"#),
399 tdef: TypeDef::bytes().infallible(),
400 }
401
402 nested_fields {
403 args: func_args![value:
404 btreemap! {
405 "log" => btreemap! {
406 "file" => btreemap! {
407 "path" => "encode_key_value.rs"
408 },
409 },
410 "agent" => btreemap! {
411 "name" => "vector",
412 "id" => 1234
413 },
414 "network" => btreemap! {
415 "ip" => value!([127, 0, 0, 1]),
416 "proto" => "tcp"
417 },
418 "event" => "log"
419 }],
420 want: Ok("agent.id=1234 agent.name=vector event=log log.file.path=encode_key_value.rs network.ip.0=127 network.ip.1=0 network.ip.2=0 network.ip.3=1 network.proto=tcp"),
421 tdef: TypeDef::bytes().infallible(),
422 }
423
424 fields_ordering {
425 args: func_args![value:
426 btreemap! {
427 "lvl" => "info",
428 "msg" => "This is a log message",
429 "log_id" => 12345,
430 },
431 fields_ordering: value!(["lvl", "msg"])
432 ],
433 want: Ok(r#"lvl=info msg="This is a log message" log_id=12345"#),
434 tdef: TypeDef::bytes().fallible(),
435 }
436
437 nested_fields_ordering {
438 args: func_args![value:
439 btreemap! {
440 "log" => btreemap! {
441 "file" => btreemap! {
442 "path" => "encode_key_value.rs"
443 },
444 },
445 "agent" => btreemap! {
446 "name" => "vector",
447 },
448 "event" => "log"
449 },
450 fields_ordering: value!(["event", "log.file.path", "agent.name"])
451 ],
452 want: Ok("event=log log.file.path=encode_key_value.rs agent.name=vector"),
453 tdef: TypeDef::bytes().fallible(),
454 }
455
456 fields_ordering_invalid_field_type {
457 args: func_args![value:
458 btreemap! {
459 "lvl" => "info",
460 "msg" => "This is a log message",
461 "log_id" => 12345,
462 },
463 fields_ordering: value!(["lvl", 2])
464 ],
465 want: Err(format!(r"invalid field value type at index 1: {}",
466 ValueError::Expected {
467 got: Kind::integer(),
468 expected: Kind::bytes()
469 })),
470 tdef: TypeDef::bytes().fallible(),
471 }
472 ];
473}