lfs: cross-repo pin refcounting on unseed, add `rad lfs backfill`
Two known gaps, both flagged as follow-up work in LFS-IPFS.md: `rad unseed` unconditionally unpinned every CID a repository's LFS notes referenced. Two repositories that happen to track byte-identical file content get the same content-addressed CID, so unseeding one could silently evict content another still-seeded repository needed -- that repo's next `git lfs fetch` would then fail with no obvious cause. Fixed by checking every other currently-seeded repository's own LFS notes before unpinning, and skipping any CID still referenced elsewhere. No persistent refcount is kept anywhere; this recomputes the still-needed set from git-notes each time, consistent with the rest of this design treating notes as the single source of truth rather than maintaining separate bookkeeping that could drift out of sync. `rad lfs backfill` retroactively pins any LFS-tracked file at HEAD that doesn't yet have a note -- the situation a `--no-verify` commit or a commit made while `ipfs`/`rad` weren't on `PATH` leaves behind (the pre-commit hook soft-skips pinning in the latter case rather than blocking the commit entirely). Content is read from Git LFS's own local object cache (populated by the clean filter on `git add` regardless of whether the hook ran), falling back to the working-tree file. Reports what it backfilled, what already had a note, and what it couldn't find content for locally (meaning some other peer that does have it needs to run this instead).
This commit is contained in:
parent
581843fa92
commit
c059c7967b
|
|
@ -1,5 +1,6 @@
|
||||||
//! `rad lfs` command implementation.
|
//! `rad lfs` command implementation.
|
||||||
|
|
||||||
|
pub mod backfill;
|
||||||
pub mod fetch;
|
pub mod fetch;
|
||||||
pub mod init;
|
pub mod init;
|
||||||
pub mod precommit;
|
pub mod precommit;
|
||||||
|
|
@ -20,5 +21,6 @@ pub fn run(args: Args, ctx: impl term::Context) -> anyhow::Result<()> {
|
||||||
Command::Fetch { oid, size, out } => self::fetch::run(oid, size, out, ctx),
|
Command::Fetch { oid, size, out } => self::fetch::run(oid, size, out, ctx),
|
||||||
Command::Rekey => self::rekey::run(ctx),
|
Command::Rekey => self::rekey::run(ctx),
|
||||||
Command::Precommit => self::precommit::run(ctx),
|
Command::Precommit => self::precommit::run(ctx),
|
||||||
|
Command::Backfill => self::backfill::run(ctx),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -51,4 +51,8 @@ pub enum Command {
|
||||||
/// "<oid> <size> <path>" lines from stdin.
|
/// "<oid> <size> <path>" lines from stdin.
|
||||||
#[command(hide = true)]
|
#[command(hide = true)]
|
||||||
Precommit,
|
Precommit,
|
||||||
|
/// Retroactively pin any LFS-tracked file at HEAD that was committed
|
||||||
|
/// without going through the pre-commit hook (e.g. `--no-verify`, or
|
||||||
|
/// the IPFS daemon/`ipfs`/`rad` weren't available at commit time)
|
||||||
|
Backfill,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,171 @@
|
||||||
|
//! `rad lfs backfill` — retroactively stores any already-committed
|
||||||
|
//! LFS-tracked file that doesn't yet have a `refs/notes/rad-lfs` note (so
|
||||||
|
//! it was never pinned to IPFS). This happens whenever the pre-commit hook
|
||||||
|
//! didn't run or didn't have what it needed at commit time: a commit made
|
||||||
|
//! with `--no-verify`, or one made while the `ipfs`/`rad` binaries weren't
|
||||||
|
//! on `PATH` yet (the hook soft-skips pinning in that case rather than
|
||||||
|
//! blocking the commit -- see `HOOK_BODY`'s `command -v` checks).
|
||||||
|
//!
|
||||||
|
//! Content actually needs to be available locally to backfill it from:
|
||||||
|
//! this reads from Git LFS's own local object cache
|
||||||
|
//! (`.git/lfs/objects/<oid prefix>/<oid>`, populated by the `git-lfs`
|
||||||
|
//! clean filter on `git add`, which -- unlike the pre-commit *hook* --
|
||||||
|
//! isn't skipped by `--no-verify`), falling back to the working-tree file
|
||||||
|
//! if that's missing. A file whose content isn't available through either
|
||||||
|
//! path can't be backfilled from this machine; some other peer that
|
||||||
|
//! already has it needs to run this instead.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use anyhow::Context as _;
|
||||||
|
|
||||||
|
use radicle::storage::{ReadRepository as _, ReadStorage as _};
|
||||||
|
|
||||||
|
use crate::ipfs;
|
||||||
|
use crate::terminal as term;
|
||||||
|
|
||||||
|
use super::store::store_object;
|
||||||
|
|
||||||
|
pub fn run(ctx: impl term::Context) -> anyhow::Result<()> {
|
||||||
|
let profile = ctx.profile()?;
|
||||||
|
let (repo, rid) = radicle::rad::cwd()
|
||||||
|
.context("`rad lfs backfill` must be run inside a Radicle repository working copy")?;
|
||||||
|
let workdir = repo
|
||||||
|
.workdir()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("`rad lfs backfill` needs a git working copy, not a bare repository"))?;
|
||||||
|
let doc = profile.storage.repository(rid)?.identity_doc()?.doc;
|
||||||
|
|
||||||
|
ipfs::check_daemon()?;
|
||||||
|
|
||||||
|
// Every path git currently tracks at HEAD (not the working tree or
|
||||||
|
// index, which may have unrelated local modifications) -- mirrors
|
||||||
|
// HOOK_BODY's own approach of shelling out to `git` for this rather
|
||||||
|
// than reimplementing tree/attribute logic against git2 directly.
|
||||||
|
let ls_files = radicle::git::run(Some(workdir), ["ls-tree", "-r", "-z", "--name-only", "HEAD"])
|
||||||
|
.context("failed to list files at HEAD")?;
|
||||||
|
let paths: Vec<String> = String::from_utf8_lossy(&ls_files.stdout)
|
||||||
|
.split('\0')
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let mut candidates = Vec::new();
|
||||||
|
for path in paths {
|
||||||
|
// Matches HOOK_BODY's own check: only files the *current*
|
||||||
|
// `.gitattributes` marks as `filter=lfs` are our concern.
|
||||||
|
let is_lfs = radicle::git::run(Some(workdir), ["check-attr", "filter", "--", &path])
|
||||||
|
.ok()
|
||||||
|
.map(|out| String::from_utf8_lossy(&out.stdout).contains(": filter: lfs"))
|
||||||
|
.unwrap_or(false);
|
||||||
|
if !is_lfs {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let blob_oid_out = radicle::git::run(Some(workdir), ["rev-parse", &format!("HEAD:{path}")])
|
||||||
|
.with_context(|| format!("failed to resolve blob oid for `{path}`"))?;
|
||||||
|
let blob_oid_str = String::from_utf8_lossy(&blob_oid_out.stdout).trim().to_string();
|
||||||
|
let Ok(blob_oid) = radicle::git::raw::Oid::from_str(&blob_oid_str) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
candidates.push((path, blob_oid));
|
||||||
|
}
|
||||||
|
|
||||||
|
if candidates.is_empty() {
|
||||||
|
term::success!("No LFS-tracked files found in HEAD; nothing to backfill");
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut signer = None;
|
||||||
|
let mut backfilled = 0;
|
||||||
|
let mut already_pinned = 0;
|
||||||
|
let mut unavailable = Vec::new();
|
||||||
|
|
||||||
|
for (path, blob_oid) in candidates {
|
||||||
|
if !crate::lfs_crypto::find_cids(&repo, blob_oid)?.is_empty() {
|
||||||
|
already_pinned += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let blob = repo
|
||||||
|
.find_blob(blob_oid)
|
||||||
|
.with_context(|| format!("failed to read blob for `{path}`"))?;
|
||||||
|
let Ok(pointer_text) = std::str::from_utf8(blob.content()) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Some(oid) = pointer_text
|
||||||
|
.lines()
|
||||||
|
.find_map(|line| line.strip_prefix("oid sha256:"))
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Some(size) = pointer_text
|
||||||
|
.lines()
|
||||||
|
.find_map(|line| line.strip_prefix("size "))
|
||||||
|
.and_then(|s| s.parse::<i64>().ok())
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
let Some(content_path) = locate_content(workdir, oid, size, &path) else {
|
||||||
|
unavailable.push(path);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
store_object(&profile, &repo, rid, &doc, oid, size, &content_path, &mut signer)
|
||||||
|
.with_context(|| format!("failed to backfill `{path}`"))?;
|
||||||
|
term::success!("Backfilled `{path}`");
|
||||||
|
backfilled += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
term::blank();
|
||||||
|
if backfilled > 0 {
|
||||||
|
term::success!("Backfilled {backfilled} object(s)");
|
||||||
|
}
|
||||||
|
if already_pinned > 0 {
|
||||||
|
term::info!("{already_pinned} object(s) already had a note; left untouched");
|
||||||
|
}
|
||||||
|
if !unavailable.is_empty() {
|
||||||
|
term::warning(format!(
|
||||||
|
"{} object(s) have no note and no content available locally to backfill from -- \
|
||||||
|
ask a peer that already has them to run `rad lfs backfill`:",
|
||||||
|
unavailable.len()
|
||||||
|
));
|
||||||
|
for path in &unavailable {
|
||||||
|
term::warning(format!(" {path}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Git LFS's own local object cache is the reliable source: the
|
||||||
|
/// `git-lfs` clean filter populates it on `git add` regardless of
|
||||||
|
/// whether the pre-commit *hook* ran (`--no-verify` skips hooks, not
|
||||||
|
/// filters). Falls back to the working-tree file, in case content is
|
||||||
|
/// present there but the LFS cache was pruned -- rejected if the size on
|
||||||
|
/// disk doesn't match what the pointer recorded, since that means it's
|
||||||
|
/// just the unsmudged pointer text, not the real content.
|
||||||
|
fn locate_content(workdir: &Path, oid: &str, size: i64, path: &str) -> Option<PathBuf> {
|
||||||
|
if oid.len() >= 4 {
|
||||||
|
let cached = workdir
|
||||||
|
.join(".git/lfs/objects")
|
||||||
|
.join(&oid[0..2])
|
||||||
|
.join(&oid[2..4])
|
||||||
|
.join(oid);
|
||||||
|
if matches_size(&cached, size) {
|
||||||
|
return Some(cached);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let working_copy = workdir.join(path);
|
||||||
|
if matches_size(&working_copy, size) {
|
||||||
|
return Some(working_copy);
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn matches_size(path: &Path, expected: i64) -> bool {
|
||||||
|
std::fs::metadata(path)
|
||||||
|
.map(|meta| meta.len() == expected as u64)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
@ -1,5 +1,8 @@
|
||||||
pub mod args;
|
pub mod args;
|
||||||
|
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
|
||||||
|
use radicle::node::policy;
|
||||||
use radicle::{Node, prelude::*};
|
use radicle::{Node, prelude::*};
|
||||||
|
|
||||||
use crate::terminal as term;
|
use crate::terminal as term;
|
||||||
|
|
@ -31,10 +34,57 @@ pub fn delete(rid: RepoId, node: &mut Node, profile: &Profile) -> anyhow::Result
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mirror the seed/unseed lifecycle for IPFS pins of this repository's
|
// Mirror the seed/unseed lifecycle for IPFS pins of this repository's
|
||||||
// Git-LFS objects.
|
// Git-LFS objects -- except any CID also referenced by an LFS note in
|
||||||
|
// some *other* still-seeded repository (e.g. two repositories that
|
||||||
|
// happen to both track a byte-identical file, giving it the same
|
||||||
|
// content-addressed CID). Unpinning unconditionally would evict
|
||||||
|
// content the other repository still needs, breaking it silently the
|
||||||
|
// next time someone there runs `git lfs fetch`. There's no
|
||||||
|
// cross-repository pin refcount kept anywhere -- this recomputes the
|
||||||
|
// still-needed set from every other seeded repository's own notes each
|
||||||
|
// time, which costs a pass over every other seeded repository but
|
||||||
|
// requires no extra persistent state, consistent with this design's
|
||||||
|
// git-notes-as-source-of-truth approach everywhere else.
|
||||||
if !cids.is_empty() {
|
if !cids.is_empty() {
|
||||||
crate::ipfs::unpin_all(&cids);
|
let still_needed = cids_still_needed_elsewhere(profile, rid)?;
|
||||||
|
let skipped = cids.iter().filter(|cid| still_needed.contains(*cid)).count();
|
||||||
|
let safe_to_unpin: Vec<String> =
|
||||||
|
cids.into_iter().filter(|cid| !still_needed.contains(cid)).collect();
|
||||||
|
|
||||||
|
if !safe_to_unpin.is_empty() {
|
||||||
|
crate::ipfs::unpin_all(&safe_to_unpin);
|
||||||
|
}
|
||||||
|
if skipped > 0 {
|
||||||
|
term::info!(
|
||||||
|
"Kept {skipped} IPFS object(s) pinned -- still referenced by another seeded repository"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The set of CIDs (among `excluding`'s own) that are also referenced by an
|
||||||
|
/// LFS note in some other repository this node is currently seeding.
|
||||||
|
fn cids_still_needed_elsewhere(
|
||||||
|
profile: &Profile,
|
||||||
|
excluding: RepoId,
|
||||||
|
) -> anyhow::Result<BTreeSet<String>> {
|
||||||
|
let mut needed = BTreeSet::new();
|
||||||
|
let store = profile.policies()?;
|
||||||
|
|
||||||
|
for entry in store.seed_policies()? {
|
||||||
|
let policy::SeedPolicy { rid, policy } = entry?;
|
||||||
|
if rid == excluding {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !matches!(policy, policy::SeedingPolicy::Allow { .. }) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Ok(repo) = profile.storage.repository(rid) {
|
||||||
|
needed.extend(crate::ipfs::lfs_cids(&repo.backend));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(needed)
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue