diff --git a/crates/radicle-cli/src/commands/lfs.rs b/crates/radicle-cli/src/commands/lfs.rs index f4eea994..73d63178 100644 --- a/crates/radicle-cli/src/commands/lfs.rs +++ b/crates/radicle-cli/src/commands/lfs.rs @@ -2,6 +2,7 @@ pub mod backfill; pub mod fetch; +pub mod fetch_batch; pub mod init; pub mod precommit; pub mod rekey; @@ -22,5 +23,6 @@ pub fn run(args: Args, ctx: impl term::Context) -> anyhow::Result<()> { Command::Rekey => self::rekey::run(ctx), Command::Precommit => self::precommit::run(ctx), Command::Backfill => self::backfill::run(ctx), + Command::FetchBatch => self::fetch_batch::run(ctx), } } diff --git a/crates/radicle-cli/src/commands/lfs/args.rs b/crates/radicle-cli/src/commands/lfs/args.rs index 6fe00716..6a2d6312 100644 --- a/crates/radicle-cli/src/commands/lfs/args.rs +++ b/crates/radicle-cli/src/commands/lfs/args.rs @@ -55,4 +55,10 @@ pub enum Command { /// without going through the pre-commit hook (e.g. `--no-verify`, or /// the IPFS daemon/`ipfs`/`rad` weren't available at commit time) Backfill, + /// Long-lived worker that fetches many LFS objects across one process + /// (invoked by the custom transfer agent; not for interactive use). + /// Reads " " lines from stdin, one at a time, + /// writing a JSON response line for each. + #[command(hide = true)] + FetchBatch, } diff --git a/crates/radicle-cli/src/commands/lfs/fetch.rs b/crates/radicle-cli/src/commands/lfs/fetch.rs index 93c5f532..a474f82f 100644 --- a/crates/radicle-cli/src/commands/lfs/fetch.rs +++ b/crates/radicle-cli/src/commands/lfs/fetch.rs @@ -3,10 +3,15 @@ //! content from the local IPFS (Kubo) node, decrypting it first if the //! repository is private. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{Context as _, anyhow}; +use radicle::Profile; +use radicle::git::raw::Repository; +use radicle::identity::RepoId; +use radicle_crypto::ssh::keystore::MemorySigner; + use crate::ipfs; use crate::lfs_crypto; use crate::terminal as term; @@ -16,13 +21,32 @@ pub fn run(oid: String, size: i64, out: PathBuf, ctx: impl term::Context) -> any let (repo, rid) = radicle::rad::cwd() .context("`rad lfs fetch` must be run inside a Radicle repository working copy")?; + let mut signer = None; + fetch_object(&profile, &repo, rid, &oid, size, &out, &mut signer) +} + +/// Fetches one LFS object's content from IPFS (decrypting it first for +/// private repos) and writes it to `out`. Factored out of [`run`] so +/// `rad lfs fetch-batch` can process many objects across a single +/// process, loading (and prompting for) the signer at most once across +/// however many objects it ends up handling, instead of once per object. +#[allow(clippy::too_many_arguments)] +pub fn fetch_object( + profile: &Profile, + repo: &Repository, + rid: RepoId, + oid: &str, + size: i64, + out: &Path, + signer: &mut Option, +) -> anyhow::Result<()> { let pointer_text = format!("version https://git-lfs.github.com/spec/v1\noid sha256:{oid}\nsize {size}\n"); let blob_oid = repo .blob(pointer_text.as_bytes()) .context("failed to write LFS pointer blob")?; - let candidates = lfs_crypto::find_envelopes(&repo, blob_oid) + let candidates = lfs_crypto::find_envelopes(repo, blob_oid) .with_context(|| format!("failed to read LFS notes for oid {oid}"))?; if candidates.is_empty() { anyhow::bail!( @@ -43,13 +67,16 @@ pub fn run(oid: String, size: i64, out: PathBuf, ctx: impl term::Context) -> any let result = match &envelope.enc { None => ipfs::cat(&envelope.cid), Some(enc) => ipfs::cat(&envelope.cid).and_then(|ciphertext| { - let signer = lfs_crypto::load_signer(&profile)?; - lfs_crypto::decrypt_with(&ciphertext, enc, &signer, profile.did(), &rid, &oid) + if signer.is_none() { + *signer = Some(lfs_crypto::load_signer(profile)?); + } + let signer = signer.as_ref().expect("just populated above"); + lfs_crypto::decrypt_with(&ciphertext, enc, signer, profile.did(), &rid, oid) }), }; match result { Ok(bytes) => { - std::fs::write(&out, bytes) + std::fs::write(out, bytes) .with_context(|| format!("failed to write `{}`", out.display()))?; return Ok(()); } diff --git a/crates/radicle-cli/src/commands/lfs/fetch_batch.rs b/crates/radicle-cli/src/commands/lfs/fetch_batch.rs new file mode 100644 index 00000000..f9151d39 --- /dev/null +++ b/crates/radicle-cli/src/commands/lfs/fetch_batch.rs @@ -0,0 +1,67 @@ +//! `rad lfs fetch-batch` — long-lived plumbing worker invoked by the Git +//! LFS custom transfer agent (`rad-lfs-transfer`) to download multiple LFS +//! objects across a single process, kept alive for the lifetime of one +//! `git lfs pull`/checkout session -- specifically so a private repo's +//! passphrase is only prompted for once per session, not once per file. +//! +//! Unlike `rad lfs precommit`/`backfill` (which know every file upfront +//! and can just read stdin until EOF), this can't collect a batch before +//! starting: git-lfs's custom-transfer protocol requests objects one at a +//! time by default (`rad-lfs-transfer` doesn't negotiate the "concurrent" +//! capability), so the caller only knows about the next object once the +//! previous one has been answered. This process instead stays alive and +//! answers requests one at a time as they arrive, reusing the same +//! unwrapped signer across all of them. +//! +//! Protocol (line-delimited, UTF-8, on stdin/stdout): +//! request: " \n" +//! response: a JSON object, one line per request, in the same order: +//! `{"ok":true}` or `{"ok":false,"error":""}` +//! The process exits cleanly on stdin EOF (the parent closing its write +//! end signals the transfer session is over). + +use std::io::{BufRead as _, Write as _}; +use std::path::PathBuf; + +use anyhow::Context as _; +use serde_json::json; + +use crate::terminal as term; + +use super::fetch::fetch_object; + +pub fn run(ctx: impl term::Context) -> anyhow::Result<()> { + let profile = ctx.profile()?; + let (repo, rid) = radicle::rad::cwd() + .context("`rad lfs fetch-batch` must be run inside a Radicle repository working copy")?; + + let mut signer = None; + let stdin = std::io::stdin(); + let mut stdout = std::io::stdout(); + + for line in stdin.lock().lines() { + let line = line.context("failed to read stdin")?; + // Each line is " " -- oid is a fixed-length + // hex string and size is decimal, so splitting on the first two + // spaces leaves the path (which may itself contain spaces) intact + // as the remainder. Mirrors `rad lfs precommit`'s input format. + let mut parts = line.splitn(3, ' '); + let (Some(oid), Some(size), Some(out)) = (parts.next(), parts.next(), parts.next()) + else { + anyhow::bail!("malformed `rad lfs fetch-batch` input line: {line:?}"); + }; + let Ok(size) = size.parse::() else { + anyhow::bail!("invalid size in `rad lfs fetch-batch` input: {size:?}"); + }; + let out = PathBuf::from(out); + + let response = match fetch_object(&profile, &repo, rid, oid, size, &out, &mut signer) { + Ok(()) => json!({"ok": true}), + Err(err) => json!({"ok": false, "error": format!("{err:#}")}), + }; + writeln!(stdout, "{response}").context("failed to write response")?; + stdout.flush().context("failed to flush stdout")?; + } + + Ok(()) +}