vdev/commands/build/
vrl_docs.rs

1use std::{
2    collections::BTreeMap,
3    fs,
4    path::{Path, PathBuf},
5};
6
7use anyhow::{Context, Result, bail};
8use serde::{Deserialize, Serialize};
9use serde_json::Value;
10
11use crate::utils::git::sparse_checkout_docs;
12use crate::utils::paths::find_repo_root;
13
14const VRL_REPO_URL: &str = "https://github.com/vectordotdev/vrl.git";
15const VECTOR_REPO_URL: &str = "https://github.com/vectordotdev/vector.git";
16const VRL_PACKAGE_NAME: &str = "vrl";
17
18/// Generate VRL function documentation by fetching pre-built JSON docs from the VRL and Vector
19/// repositories.
20///
21/// VRL stdlib docs come from the VRL repo (`docs/generated/*.json`), and Vector-specific function
22/// docs come from the Vector repo (`docs/generated/*.json`). Both sets are merged into a single
23/// `generated.cue` output file.
24#[derive(clap::Args, Debug)]
25#[command()]
26pub struct Cli {
27    /// Output directory for the generated.cue file
28    #[arg(short, long)]
29    output_dir: PathBuf,
30
31    /// VRL commit SHA to fetch docs from. If unspecified, read from Cargo.lock.
32    #[arg(long)]
33    vrl_sha: Option<String>,
34
35    /// Vector commit SHA to fetch docs from. If unspecified, read docs/generated locally.
36    #[arg(long)]
37    vector_sha: Option<String>,
38}
39
40#[derive(Serialize)]
41struct FunctionDocWrapper {
42    remap: RemapWrapper,
43}
44
45#[derive(Serialize)]
46struct RemapWrapper {
47    functions: BTreeMap<String, Value>,
48}
49
50impl Cli {
51    pub fn exec(self) -> Result<()> {
52        let repo_root = find_repo_root()?;
53        let temp_dir = tempfile::tempdir().context("Failed to create temp directory")?;
54
55        // VRL stdlib docs
56        let vrl_sha = match self.vrl_sha {
57            Some(sha) => sha,
58            None => get_vrl_commit_sha(&repo_root)?,
59        };
60        info!("VRL commit SHA: {vrl_sha}");
61
62        let vrl_clone_dir = temp_dir.path().join("vrl");
63        sparse_checkout_docs(&vrl_sha, VRL_REPO_URL, &vrl_clone_dir)?;
64        let vrl_docs_dir = vrl_clone_dir.join("docs").join("generated");
65
66        let mut functions = read_function_docs(&vrl_docs_dir)?;
67        info!("Read {} VRL stdlib function docs", functions.len());
68
69        // Vector-specific docs
70        let vector_docs_dir = if let Some(vector_sha) = &self.vector_sha {
71            info!("Vector commit SHA: {vector_sha}");
72            let vector_clone_dir = temp_dir.path().join("vector");
73            sparse_checkout_docs(vector_sha, VECTOR_REPO_URL, &vector_clone_dir)?;
74            vector_clone_dir.join("docs").join("generated")
75        } else {
76            repo_root.join("docs").join("generated")
77        };
78
79        let vector_functions = read_function_docs(&vector_docs_dir)?;
80        info!("Read {} Vector function docs", vector_functions.len());
81        functions.extend(vector_functions);
82
83        let wrapper = FunctionDocWrapper {
84            remap: RemapWrapper { functions },
85        };
86
87        fs::create_dir_all(&self.output_dir)?;
88        let mut json = serde_json::to_string(&wrapper)?;
89        json.push('\n');
90        let filepath = self.output_dir.join("generated.cue");
91        fs::write(&filepath, json)?;
92
93        info!("Generated: {}", filepath.display());
94        Ok(())
95    }
96}
97
98/// A minimal representation of a `[[package]]` entry in `Cargo.lock`.
99#[derive(Deserialize)]
100struct LockPackage {
101    name: String,
102    version: String,
103    source: Option<String>,
104}
105
106#[derive(Deserialize)]
107struct CargoLock {
108    package: Vec<LockPackage>,
109}
110
111/// Parse `Cargo.lock` to find a git ref for the `vrl` package.
112///
113/// Returns the commit SHA for git-sourced dependencies, or a version tag (e.g. `v0.31.0`) for
114/// registry-sourced dependencies.
115fn get_vrl_commit_sha(repo_root: &Path) -> Result<String> {
116    let lock_path = repo_root.join("Cargo.lock");
117    let lock_text = fs::read_to_string(&lock_path)
118        .with_context(|| format!("Failed to read {}", lock_path.display()))?;
119
120    let lock: CargoLock =
121        toml::from_str(&lock_text).context("Failed to parse Cargo.lock as TOML")?;
122
123    let pkg = lock
124        .package
125        .iter()
126        .find(|p| p.name == VRL_PACKAGE_NAME)
127        .context("Could not find VRL package in Cargo.lock")?;
128
129    match pkg.source.as_deref() {
130        // Git source: "git+https://github.com/vectordotdev/vrl.git?branch=main#5316c01b..."
131        Some(source) if source.starts_with("git+") => source
132            .rsplit_once('#')
133            .map(|(_, sha)| sha.to_string())
134            .context("Could not extract commit SHA from VRL git source string"),
135        // Registry source (crates.io): use the version as a tag
136        Some(source) if source.starts_with("registry+") => Ok(format!("v{}", pkg.version)),
137        Some(source) => bail!("Unrecognized VRL package source in Cargo.lock: {source}"),
138        None => bail!("VRL package in Cargo.lock has no source field"),
139    }
140}
141
142/// Read all `*.json` files from a directory into a name->value map.
143fn read_function_docs(docs_dir: &Path) -> Result<BTreeMap<String, Value>> {
144    let mut functions = BTreeMap::new();
145
146    let entries: Vec<_> = fs::read_dir(docs_dir)
147        .with_context(|| format!("Failed to read docs directory: {}", docs_dir.display()))?
148        .collect::<Result<Vec<_>, _>>()
149        .context("Failed to iterate docs directory")?;
150
151    for entry in entries {
152        let path = entry.path();
153        if path.extension().is_some_and(|ext| ext == "json") {
154            let content = fs::read_to_string(&path)
155                .with_context(|| format!("Failed to read {}", path.display()))?;
156            let value: Value = serde_json::from_str(&content)
157                .with_context(|| format!("Failed to parse JSON from {}", path.display()))?;
158
159            let name = path
160                .file_stem()
161                .and_then(|s| s.to_str())
162                .context("Invalid filename")?
163                .to_string();
164
165            functions.insert(name, value);
166        }
167    }
168
169    Ok(functions)
170}
vdev/commands/build/vrl_docs.rs

vdev/commands/build/
vrl_docs.rs
