vrl/stdlib/
merge.rs

1use crate::compiler::prelude::*;
2use std::collections::BTreeMap;
3use std::sync::LazyLock;
4
5static DEFAULT_DEEP: LazyLock<Value> = LazyLock::new(|| Value::Boolean(false));
6
7static PARAMETERS: LazyLock<Vec<Parameter>> = LazyLock::new(|| {
8    vec![
9        Parameter::required("to", kind::OBJECT, "The object to merge into."),
10        Parameter::required("from", kind::OBJECT, "The object to merge from."),
11        Parameter::optional(
12            "deep",
13            kind::BOOLEAN,
14            "A deep merge is performed if `true`, otherwise only top-level fields are merged.",
15        )
16        .default(&DEFAULT_DEEP),
17    ]
18});
19
20#[derive(Clone, Copy, Debug)]
21pub struct Merge;
22
23impl Function for Merge {
24    fn identifier(&self) -> &'static str {
25        "merge"
26    }
27
28    fn usage(&self) -> &'static str {
29        "Merges the `from` object into the `to` object."
30    }
31
32    fn category(&self) -> &'static str {
33        Category::Object.as_ref()
34    }
35
36    fn return_kind(&self) -> u16 {
37        kind::OBJECT
38    }
39
40    fn return_rules(&self) -> &'static [&'static str] {
41        &[
42            "The field from the `from` object is chosen if a key exists in both objects.",
43            "Objects are merged recursively if `deep` is specified, a key exists in both objects, and both of those
44fields are also objects.",
45        ]
46    }
47
48    fn parameters(&self) -> &'static [Parameter] {
49        PARAMETERS.as_slice()
50    }
51
52    fn examples(&self) -> &'static [Example] {
53        &[
54            example! {
55                title: "Object merge (shallow)",
56                source: indoc! {r#"
57                    merge(
58                        {
59                            "parent1": {
60                                "child1": 1,
61                                "child2": 2
62                            },
63                            "parent2": {
64                                "child3": 3
65                            }
66                        },
67                        {
68                            "parent1": {
69                                "child2": 4,
70                                "child5": 5
71                            }
72                        }
73                    )
74                "#},
75                result: Ok(r#"{ "parent1": { "child2": 4, "child5": 5 }, "parent2": { "child3": 3 } }"#),
76            },
77            example! {
78                title: "Object merge (deep)",
79                source: indoc! {r#"
80                    merge(
81                        {
82                            "parent1": {
83                                "child1": 1,
84                                "child2": 2
85                            },
86                            "parent2": {
87                                "child3": 3
88                            }
89                        },
90                        {
91                            "parent1": {
92                                "child2": 4,
93                                "child5": 5
94                            }
95                        },
96                        deep: true
97                    )
98                "#},
99                result: Ok(r#"{ "parent1": { "child1": 1, "child2": 4, "child5": 5 }, "parent2": { "child3": 3 } }"#),
100            },
101        ]
102    }
103
104    fn compile(
105        &self,
106        _state: &state::TypeState,
107        _ctx: &mut FunctionCompileContext,
108        arguments: ArgumentList,
109    ) -> Compiled {
110        let to = arguments.required("to");
111        let from = arguments.required("from");
112        let deep = arguments.optional("deep");
113
114        Ok(MergeFn { to, from, deep }.as_expr())
115    }
116}
117
118#[derive(Debug, Clone)]
119pub(crate) struct MergeFn {
120    to: Box<dyn Expression>,
121    from: Box<dyn Expression>,
122    deep: Option<Box<dyn Expression>>,
123}
124
125impl FunctionExpression for MergeFn {
126    fn resolve(&self, ctx: &mut Context) -> Resolved {
127        let mut to_value = self.to.resolve(ctx)?.try_object()?;
128        let from_value = self.from.resolve(ctx)?.try_object()?;
129        let deep = self
130            .deep
131            .map_resolve_with_default(ctx, || DEFAULT_DEEP.clone())?
132            .try_boolean()?;
133
134        merge_maps(&mut to_value, &from_value, deep);
135
136        Ok(to_value.into())
137    }
138
139    fn type_def(&self, state: &state::TypeState) -> TypeDef {
140        // TODO: this has a known bug when deep is true
141        // see: https://github.com/vectordotdev/vector/issues/13597
142        self.to
143            .type_def(state)
144            .restrict_object()
145            .merge_overwrite(self.from.type_def(state).restrict_object())
146    }
147}
148
149/// Merges two `BTreeMaps` of Symbol’s value as variable is void: Values. The
150/// second map is merged into the first one.
151///
152/// If Symbol’s value as variable is void: deep is true, only the top level
153/// values are merged in. If both maps contain a field with the same name, the
154/// field from the first is overwritten with the field from the second.
155///
156/// If Symbol’s value as variable is void: deep is false, should both maps
157/// contain a field with the same name, and both those fields are also maps, the
158/// function will recurse and will merge the child fields from the second into
159/// the child fields from the first.
160///
161/// Note, this does recurse, so there is the theoretical possibility that it
162/// could blow up the stack. From quick tests on a sample project I was able to
163/// merge maps with a depth of 3,500 before encountering issues. So I think that
164/// is likely to be within acceptable limits. If it becomes a problem, we can
165/// unroll this function, but that will come at a cost of extra code complexity.
166fn merge_maps<K>(map1: &mut BTreeMap<K, Value>, map2: &BTreeMap<K, Value>, deep: bool)
167where
168    K: std::cmp::Ord + Clone,
169{
170    for (key2, value2) in map2 {
171        match (deep, map1.get_mut(key2), value2) {
172            (true, Some(Value::Object(child1)), Value::Object(child2)) => {
173                // We are doing a deep merge and both fields are maps.
174                merge_maps(child1, child2, deep);
175            }
176            _ => {
177                map1.insert(key2.clone(), value2.clone());
178            }
179        }
180    }
181}
182
183#[cfg(test)]
184mod tests {
185    use super::*;
186    use crate::{btreemap, value};
187
188    test_function! [
189        merge => Merge;
190
191        simple {
192            args: func_args![
193                to: value!({ key1: "val1" }),
194                from: value!({ key2: "val2" })
195            ],
196            want: Ok(value!({ key1: "val1", key2: "val2" })),
197            tdef: TypeDef::object(btreemap! {
198                Field::from("key1") => Kind::bytes(),
199                Field::from("key2") => Kind::bytes(),
200            }),
201        }
202
203        shallow {
204            args: func_args![
205                to: value!({
206                    key1: "val1",
207                    child: { grandchild1: "val1" },
208                }),
209                from: value!({
210                    key2: "val2",
211                    child: { grandchild2: true },
212                })
213            ],
214            want: Ok(value!({
215                key1: "val1",
216                key2: "val2",
217                child: { grandchild2: true },
218            })),
219            tdef: TypeDef::object(btreemap! {
220                Field::from("key1") => Kind::bytes(),
221                Field::from("key2") => Kind::bytes(),
222                Field::from("child") => TypeDef::object(btreemap! {
223                    Field::from("grandchild2") => Kind::boolean(),
224                }),
225            }),
226        }
227
228        deep {
229            args: func_args![
230                to: value!({
231                    key1: "val1",
232                    child: { grandchild1: "val1" },
233                }),
234                from: value!({
235                    key2: "val2",
236                    child: { grandchild2: true },
237                }),
238                deep: true,
239            ],
240            want: Ok(value!({
241                key1: "val1",
242                key2: "val2",
243                child: {
244                    grandchild1: "val1",
245                    grandchild2: true,
246                },
247            })),
248            tdef: TypeDef::object(btreemap! {
249                Field::from("key1") => Kind::bytes(),
250                Field::from("key2") => Kind::bytes(),
251                Field::from("child") => TypeDef::object(btreemap! {
252                    Field::from("grandchild2") => Kind::boolean(),
253                }),
254            }),
255
256        }
257    ];
258}
vrl/stdlib/merge.rs

vrl/stdlib/
merge.rs
