lfs: cross-repo pin refcounting on unseed, add `rad lfs backfill`

Two known gaps, both flagged as follow-up work in LFS-IPFS.md:

`rad unseed` unconditionally unpinned every CID a repository's LFS
notes referenced. Two repositories that happen to track
byte-identical file content get the same content-addressed CID, so
unseeding one could silently evict content another still-seeded
repository needed -- that repo's next `git lfs fetch` would then fail
with no obvious cause. Fixed by checking every other currently-seeded
repository's own LFS notes before unpinning, and skipping any CID
still referenced elsewhere. No persistent refcount is kept anywhere;
this recomputes the still-needed set from git-notes each time,
consistent with the rest of this design treating notes as the single
source of truth rather than maintaining separate bookkeeping that
could drift out of sync.

`rad lfs backfill` retroactively pins any LFS-tracked file at HEAD
that doesn't yet have a note -- the situation a `--no-verify` commit
or a commit made while `ipfs`/`rad` weren't on `PATH` leaves behind
(the pre-commit hook soft-skips pinning in the latter case rather
than blocking the commit entirely). Content is read from Git LFS's
own local object cache (populated by the clean filter on `git add`
regardless of whether the hook ran), falling back to the working-tree
file. Reports what it backfilled, what already had a note, and what
it couldn't find content for locally (meaning some other peer that
does have it needs to run this instead).
This commit is contained in:
Maciek "mab122" Bator 2026-07-17 15:21:40 +02:00
parent 581843fa92
commit c059c7967b
4 changed files with 229 additions and 2 deletions

View File

@ -1,5 +1,6 @@
//! `rad lfs` command implementation.
pub mod backfill;
pub mod fetch;
pub mod init;
pub mod precommit;
@ -20,5 +21,6 @@ pub fn run(args: Args, ctx: impl term::Context) -> anyhow::Result<()> {
Command::Fetch { oid, size, out } => self::fetch::run(oid, size, out, ctx),
Command::Rekey => self::rekey::run(ctx),
Command::Precommit => self::precommit::run(ctx),
Command::Backfill => self::backfill::run(ctx),
}
}

View File

@ -51,4 +51,8 @@ pub enum Command {
/// "<oid> <size> <path>" lines from stdin.
#[command(hide = true)]
Precommit,
/// Retroactively pin any LFS-tracked file at HEAD that was committed
/// without going through the pre-commit hook (e.g. `--no-verify`, or
/// the IPFS daemon/`ipfs`/`rad` weren't available at commit time)
Backfill,
}

View File

@ -0,0 +1,171 @@
//! `rad lfs backfill` — retroactively stores any already-committed
//! LFS-tracked file that doesn't yet have a `refs/notes/rad-lfs` note (so
//! it was never pinned to IPFS). This happens whenever the pre-commit hook
//! didn't run or didn't have what it needed at commit time: a commit made
//! with `--no-verify`, or one made while the `ipfs`/`rad` binaries weren't
//! on `PATH` yet (the hook soft-skips pinning in that case rather than
//! blocking the commit -- see `HOOK_BODY`'s `command -v` checks).
//!
//! Content actually needs to be available locally to backfill it from:
//! this reads from Git LFS's own local object cache
//! (`.git/lfs/objects/<oid prefix>/<oid>`, populated by the `git-lfs`
//! clean filter on `git add`, which -- unlike the pre-commit *hook* --
//! isn't skipped by `--no-verify`), falling back to the working-tree file
//! if that's missing. A file whose content isn't available through either
//! path can't be backfilled from this machine; some other peer that
//! already has it needs to run this instead.
use std::path::{Path, PathBuf};
use anyhow::Context as _;
use radicle::storage::{ReadRepository as _, ReadStorage as _};
use crate::ipfs;
use crate::terminal as term;
use super::store::store_object;
pub fn run(ctx: impl term::Context) -> anyhow::Result<()> {
let profile = ctx.profile()?;
let (repo, rid) = radicle::rad::cwd()
.context("`rad lfs backfill` must be run inside a Radicle repository working copy")?;
let workdir = repo
.workdir()
.ok_or_else(|| anyhow::anyhow!("`rad lfs backfill` needs a git working copy, not a bare repository"))?;
let doc = profile.storage.repository(rid)?.identity_doc()?.doc;
ipfs::check_daemon()?;
// Every path git currently tracks at HEAD (not the working tree or
// index, which may have unrelated local modifications) -- mirrors
// HOOK_BODY's own approach of shelling out to `git` for this rather
// than reimplementing tree/attribute logic against git2 directly.
let ls_files = radicle::git::run(Some(workdir), ["ls-tree", "-r", "-z", "--name-only", "HEAD"])
.context("failed to list files at HEAD")?;
let paths: Vec<String> = String::from_utf8_lossy(&ls_files.stdout)
.split('\0')
.filter(|s| !s.is_empty())
.map(str::to_string)
.collect();
let mut candidates = Vec::new();
for path in paths {
// Matches HOOK_BODY's own check: only files the *current*
// `.gitattributes` marks as `filter=lfs` are our concern.
let is_lfs = radicle::git::run(Some(workdir), ["check-attr", "filter", "--", &path])
.ok()
.map(|out| String::from_utf8_lossy(&out.stdout).contains(": filter: lfs"))
.unwrap_or(false);
if !is_lfs {
continue;
}
let blob_oid_out = radicle::git::run(Some(workdir), ["rev-parse", &format!("HEAD:{path}")])
.with_context(|| format!("failed to resolve blob oid for `{path}`"))?;
let blob_oid_str = String::from_utf8_lossy(&blob_oid_out.stdout).trim().to_string();
let Ok(blob_oid) = radicle::git::raw::Oid::from_str(&blob_oid_str) else {
continue;
};
candidates.push((path, blob_oid));
}
if candidates.is_empty() {
term::success!("No LFS-tracked files found in HEAD; nothing to backfill");
return Ok(());
}
let mut signer = None;
let mut backfilled = 0;
let mut already_pinned = 0;
let mut unavailable = Vec::new();
for (path, blob_oid) in candidates {
if !crate::lfs_crypto::find_cids(&repo, blob_oid)?.is_empty() {
already_pinned += 1;
continue;
}
let blob = repo
.find_blob(blob_oid)
.with_context(|| format!("failed to read blob for `{path}`"))?;
let Ok(pointer_text) = std::str::from_utf8(blob.content()) else {
continue;
};
let Some(oid) = pointer_text
.lines()
.find_map(|line| line.strip_prefix("oid sha256:"))
else {
continue;
};
let Some(size) = pointer_text
.lines()
.find_map(|line| line.strip_prefix("size "))
.and_then(|s| s.parse::<i64>().ok())
else {
continue;
};
let Some(content_path) = locate_content(workdir, oid, size, &path) else {
unavailable.push(path);
continue;
};
store_object(&profile, &repo, rid, &doc, oid, size, &content_path, &mut signer)
.with_context(|| format!("failed to backfill `{path}`"))?;
term::success!("Backfilled `{path}`");
backfilled += 1;
}
term::blank();
if backfilled > 0 {
term::success!("Backfilled {backfilled} object(s)");
}
if already_pinned > 0 {
term::info!("{already_pinned} object(s) already had a note; left untouched");
}
if !unavailable.is_empty() {
term::warning(format!(
"{} object(s) have no note and no content available locally to backfill from -- \
ask a peer that already has them to run `rad lfs backfill`:",
unavailable.len()
));
for path in &unavailable {
term::warning(format!(" {path}"));
}
}
Ok(())
}
/// Git LFS's own local object cache is the reliable source: the
/// `git-lfs` clean filter populates it on `git add` regardless of
/// whether the pre-commit *hook* ran (`--no-verify` skips hooks, not
/// filters). Falls back to the working-tree file, in case content is
/// present there but the LFS cache was pruned -- rejected if the size on
/// disk doesn't match what the pointer recorded, since that means it's
/// just the unsmudged pointer text, not the real content.
fn locate_content(workdir: &Path, oid: &str, size: i64, path: &str) -> Option<PathBuf> {
if oid.len() >= 4 {
let cached = workdir
.join(".git/lfs/objects")
.join(&oid[0..2])
.join(&oid[2..4])
.join(oid);
if matches_size(&cached, size) {
return Some(cached);
}
}
let working_copy = workdir.join(path);
if matches_size(&working_copy, size) {
return Some(working_copy);
}
None
}
fn matches_size(path: &Path, expected: i64) -> bool {
std::fs::metadata(path)
.map(|meta| meta.len() == expected as u64)
.unwrap_or(false)
}

View File

@ -1,5 +1,8 @@
pub mod args;
use std::collections::BTreeSet;
use radicle::node::policy;
use radicle::{Node, prelude::*};
use crate::terminal as term;
@ -31,10 +34,57 @@ pub fn delete(rid: RepoId, node: &mut Node, profile: &Profile) -> anyhow::Result
}
// Mirror the seed/unseed lifecycle for IPFS pins of this repository's
// Git-LFS objects.
// Git-LFS objects -- except any CID also referenced by an LFS note in
// some *other* still-seeded repository (e.g. two repositories that
// happen to both track a byte-identical file, giving it the same
// content-addressed CID). Unpinning unconditionally would evict
// content the other repository still needs, breaking it silently the
// next time someone there runs `git lfs fetch`. There's no
// cross-repository pin refcount kept anywhere -- this recomputes the
// still-needed set from every other seeded repository's own notes each
// time, which costs a pass over every other seeded repository but
// requires no extra persistent state, consistent with this design's
// git-notes-as-source-of-truth approach everywhere else.
if !cids.is_empty() {
crate::ipfs::unpin_all(&cids);
let still_needed = cids_still_needed_elsewhere(profile, rid)?;
let skipped = cids.iter().filter(|cid| still_needed.contains(*cid)).count();
let safe_to_unpin: Vec<String> =
cids.into_iter().filter(|cid| !still_needed.contains(cid)).collect();
if !safe_to_unpin.is_empty() {
crate::ipfs::unpin_all(&safe_to_unpin);
}
if skipped > 0 {
term::info!(
"Kept {skipped} IPFS object(s) pinned -- still referenced by another seeded repository"
);
}
}
Ok(())
}
/// The set of CIDs (among `excluding`'s own) that are also referenced by an
/// LFS note in some other repository this node is currently seeding.
fn cids_still_needed_elsewhere(
profile: &Profile,
excluding: RepoId,
) -> anyhow::Result<BTreeSet<String>> {
let mut needed = BTreeSet::new();
let store = profile.policies()?;
for entry in store.seed_policies()? {
let policy::SeedPolicy { rid, policy } = entry?;
if rid == excluding {
continue;
}
if !matches!(policy, policy::SeedingPolicy::Allow { .. }) {
continue;
}
if let Ok(repo) = profile.storage.repository(rid) {
needed.extend(crate::ipfs::lfs_cids(&repo.backend));
}
}
Ok(needed)
}