From c059c7967bb086f14316924ac10265148815d28b Mon Sep 17 00:00:00 2001 From: "Maciek \"mab122\" Bator" Date: Fri, 17 Jul 2026 15:21:40 +0200 Subject: [PATCH] lfs: cross-repo pin refcounting on unseed, add `rad lfs backfill` Two known gaps, both flagged as follow-up work in LFS-IPFS.md: `rad unseed` unconditionally unpinned every CID a repository's LFS notes referenced. Two repositories that happen to track byte-identical file content get the same content-addressed CID, so unseeding one could silently evict content another still-seeded repository needed -- that repo's next `git lfs fetch` would then fail with no obvious cause. Fixed by checking every other currently-seeded repository's own LFS notes before unpinning, and skipping any CID still referenced elsewhere. No persistent refcount is kept anywhere; this recomputes the still-needed set from git-notes each time, consistent with the rest of this design treating notes as the single source of truth rather than maintaining separate bookkeeping that could drift out of sync. `rad lfs backfill` retroactively pins any LFS-tracked file at HEAD that doesn't yet have a note -- the situation a `--no-verify` commit or a commit made while `ipfs`/`rad` weren't on `PATH` leaves behind (the pre-commit hook soft-skips pinning in the latter case rather than blocking the commit entirely). Content is read from Git LFS's own local object cache (populated by the clean filter on `git add` regardless of whether the hook ran), falling back to the working-tree file. Reports what it backfilled, what already had a note, and what it couldn't find content for locally (meaning some other peer that does have it needs to run this instead). --- crates/radicle-cli/src/commands/lfs.rs | 2 + crates/radicle-cli/src/commands/lfs/args.rs | 4 + .../radicle-cli/src/commands/lfs/backfill.rs | 171 ++++++++++++++++++ crates/radicle-cli/src/commands/unseed.rs | 54 +++++- 4 files changed, 229 insertions(+), 2 deletions(-) create mode 100644 crates/radicle-cli/src/commands/lfs/backfill.rs diff --git a/crates/radicle-cli/src/commands/lfs.rs b/crates/radicle-cli/src/commands/lfs.rs index 373d9d42..f4eea994 100644 --- a/crates/radicle-cli/src/commands/lfs.rs +++ b/crates/radicle-cli/src/commands/lfs.rs @@ -1,5 +1,6 @@ //! `rad lfs` command implementation. +pub mod backfill; pub mod fetch; pub mod init; pub mod precommit; @@ -20,5 +21,6 @@ pub fn run(args: Args, ctx: impl term::Context) -> anyhow::Result<()> { Command::Fetch { oid, size, out } => self::fetch::run(oid, size, out, ctx), Command::Rekey => self::rekey::run(ctx), Command::Precommit => self::precommit::run(ctx), + Command::Backfill => self::backfill::run(ctx), } } diff --git a/crates/radicle-cli/src/commands/lfs/args.rs b/crates/radicle-cli/src/commands/lfs/args.rs index 6b8feaad..6fe00716 100644 --- a/crates/radicle-cli/src/commands/lfs/args.rs +++ b/crates/radicle-cli/src/commands/lfs/args.rs @@ -51,4 +51,8 @@ pub enum Command { /// " " lines from stdin. #[command(hide = true)] Precommit, + /// Retroactively pin any LFS-tracked file at HEAD that was committed + /// without going through the pre-commit hook (e.g. `--no-verify`, or + /// the IPFS daemon/`ipfs`/`rad` weren't available at commit time) + Backfill, } diff --git a/crates/radicle-cli/src/commands/lfs/backfill.rs b/crates/radicle-cli/src/commands/lfs/backfill.rs new file mode 100644 index 00000000..c59bc1e6 --- /dev/null +++ b/crates/radicle-cli/src/commands/lfs/backfill.rs @@ -0,0 +1,171 @@ +//! `rad lfs backfill` — retroactively stores any already-committed +//! LFS-tracked file that doesn't yet have a `refs/notes/rad-lfs` note (so +//! it was never pinned to IPFS). This happens whenever the pre-commit hook +//! didn't run or didn't have what it needed at commit time: a commit made +//! with `--no-verify`, or one made while the `ipfs`/`rad` binaries weren't +//! on `PATH` yet (the hook soft-skips pinning in that case rather than +//! blocking the commit -- see `HOOK_BODY`'s `command -v` checks). +//! +//! Content actually needs to be available locally to backfill it from: +//! this reads from Git LFS's own local object cache +//! (`.git/lfs/objects//`, populated by the `git-lfs` +//! clean filter on `git add`, which -- unlike the pre-commit *hook* -- +//! isn't skipped by `--no-verify`), falling back to the working-tree file +//! if that's missing. A file whose content isn't available through either +//! path can't be backfilled from this machine; some other peer that +//! already has it needs to run this instead. + +use std::path::{Path, PathBuf}; + +use anyhow::Context as _; + +use radicle::storage::{ReadRepository as _, ReadStorage as _}; + +use crate::ipfs; +use crate::terminal as term; + +use super::store::store_object; + +pub fn run(ctx: impl term::Context) -> anyhow::Result<()> { + let profile = ctx.profile()?; + let (repo, rid) = radicle::rad::cwd() + .context("`rad lfs backfill` must be run inside a Radicle repository working copy")?; + let workdir = repo + .workdir() + .ok_or_else(|| anyhow::anyhow!("`rad lfs backfill` needs a git working copy, not a bare repository"))?; + let doc = profile.storage.repository(rid)?.identity_doc()?.doc; + + ipfs::check_daemon()?; + + // Every path git currently tracks at HEAD (not the working tree or + // index, which may have unrelated local modifications) -- mirrors + // HOOK_BODY's own approach of shelling out to `git` for this rather + // than reimplementing tree/attribute logic against git2 directly. + let ls_files = radicle::git::run(Some(workdir), ["ls-tree", "-r", "-z", "--name-only", "HEAD"]) + .context("failed to list files at HEAD")?; + let paths: Vec = String::from_utf8_lossy(&ls_files.stdout) + .split('\0') + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect(); + + let mut candidates = Vec::new(); + for path in paths { + // Matches HOOK_BODY's own check: only files the *current* + // `.gitattributes` marks as `filter=lfs` are our concern. + let is_lfs = radicle::git::run(Some(workdir), ["check-attr", "filter", "--", &path]) + .ok() + .map(|out| String::from_utf8_lossy(&out.stdout).contains(": filter: lfs")) + .unwrap_or(false); + if !is_lfs { + continue; + } + let blob_oid_out = radicle::git::run(Some(workdir), ["rev-parse", &format!("HEAD:{path}")]) + .with_context(|| format!("failed to resolve blob oid for `{path}`"))?; + let blob_oid_str = String::from_utf8_lossy(&blob_oid_out.stdout).trim().to_string(); + let Ok(blob_oid) = radicle::git::raw::Oid::from_str(&blob_oid_str) else { + continue; + }; + candidates.push((path, blob_oid)); + } + + if candidates.is_empty() { + term::success!("No LFS-tracked files found in HEAD; nothing to backfill"); + return Ok(()); + } + + let mut signer = None; + let mut backfilled = 0; + let mut already_pinned = 0; + let mut unavailable = Vec::new(); + + for (path, blob_oid) in candidates { + if !crate::lfs_crypto::find_cids(&repo, blob_oid)?.is_empty() { + already_pinned += 1; + continue; + } + + let blob = repo + .find_blob(blob_oid) + .with_context(|| format!("failed to read blob for `{path}`"))?; + let Ok(pointer_text) = std::str::from_utf8(blob.content()) else { + continue; + }; + let Some(oid) = pointer_text + .lines() + .find_map(|line| line.strip_prefix("oid sha256:")) + else { + continue; + }; + let Some(size) = pointer_text + .lines() + .find_map(|line| line.strip_prefix("size ")) + .and_then(|s| s.parse::().ok()) + else { + continue; + }; + + let Some(content_path) = locate_content(workdir, oid, size, &path) else { + unavailable.push(path); + continue; + }; + + store_object(&profile, &repo, rid, &doc, oid, size, &content_path, &mut signer) + .with_context(|| format!("failed to backfill `{path}`"))?; + term::success!("Backfilled `{path}`"); + backfilled += 1; + } + + term::blank(); + if backfilled > 0 { + term::success!("Backfilled {backfilled} object(s)"); + } + if already_pinned > 0 { + term::info!("{already_pinned} object(s) already had a note; left untouched"); + } + if !unavailable.is_empty() { + term::warning(format!( + "{} object(s) have no note and no content available locally to backfill from -- \ + ask a peer that already has them to run `rad lfs backfill`:", + unavailable.len() + )); + for path in &unavailable { + term::warning(format!(" {path}")); + } + } + + Ok(()) +} + +/// Git LFS's own local object cache is the reliable source: the +/// `git-lfs` clean filter populates it on `git add` regardless of +/// whether the pre-commit *hook* ran (`--no-verify` skips hooks, not +/// filters). Falls back to the working-tree file, in case content is +/// present there but the LFS cache was pruned -- rejected if the size on +/// disk doesn't match what the pointer recorded, since that means it's +/// just the unsmudged pointer text, not the real content. +fn locate_content(workdir: &Path, oid: &str, size: i64, path: &str) -> Option { + if oid.len() >= 4 { + let cached = workdir + .join(".git/lfs/objects") + .join(&oid[0..2]) + .join(&oid[2..4]) + .join(oid); + if matches_size(&cached, size) { + return Some(cached); + } + } + + let working_copy = workdir.join(path); + if matches_size(&working_copy, size) { + return Some(working_copy); + } + + None +} + +fn matches_size(path: &Path, expected: i64) -> bool { + std::fs::metadata(path) + .map(|meta| meta.len() == expected as u64) + .unwrap_or(false) +} diff --git a/crates/radicle-cli/src/commands/unseed.rs b/crates/radicle-cli/src/commands/unseed.rs index 5ef2b4ce..8eb675f9 100644 --- a/crates/radicle-cli/src/commands/unseed.rs +++ b/crates/radicle-cli/src/commands/unseed.rs @@ -1,5 +1,8 @@ pub mod args; +use std::collections::BTreeSet; + +use radicle::node::policy; use radicle::{Node, prelude::*}; use crate::terminal as term; @@ -31,10 +34,57 @@ pub fn delete(rid: RepoId, node: &mut Node, profile: &Profile) -> anyhow::Result } // Mirror the seed/unseed lifecycle for IPFS pins of this repository's - // Git-LFS objects. + // Git-LFS objects -- except any CID also referenced by an LFS note in + // some *other* still-seeded repository (e.g. two repositories that + // happen to both track a byte-identical file, giving it the same + // content-addressed CID). Unpinning unconditionally would evict + // content the other repository still needs, breaking it silently the + // next time someone there runs `git lfs fetch`. There's no + // cross-repository pin refcount kept anywhere -- this recomputes the + // still-needed set from every other seeded repository's own notes each + // time, which costs a pass over every other seeded repository but + // requires no extra persistent state, consistent with this design's + // git-notes-as-source-of-truth approach everywhere else. if !cids.is_empty() { - crate::ipfs::unpin_all(&cids); + let still_needed = cids_still_needed_elsewhere(profile, rid)?; + let skipped = cids.iter().filter(|cid| still_needed.contains(*cid)).count(); + let safe_to_unpin: Vec = + cids.into_iter().filter(|cid| !still_needed.contains(cid)).collect(); + + if !safe_to_unpin.is_empty() { + crate::ipfs::unpin_all(&safe_to_unpin); + } + if skipped > 0 { + term::info!( + "Kept {skipped} IPFS object(s) pinned -- still referenced by another seeded repository" + ); + } } Ok(()) } + +/// The set of CIDs (among `excluding`'s own) that are also referenced by an +/// LFS note in some other repository this node is currently seeding. +fn cids_still_needed_elsewhere( + profile: &Profile, + excluding: RepoId, +) -> anyhow::Result> { + let mut needed = BTreeSet::new(); + let store = profile.policies()?; + + for entry in store.seed_policies()? { + let policy::SeedPolicy { rid, policy } = entry?; + if rid == excluding { + continue; + } + if !matches!(policy, policy::SeedingPolicy::Allow { .. }) { + continue; + } + if let Ok(repo) = profile.storage.repository(rid) { + needed.extend(crate::ipfs::lfs_cids(&repo.backend)); + } + } + + Ok(needed) +}