diff --git a/README.md b/README.md index b648ca4..33e3ba5 100644 --- a/README.md +++ b/README.md @@ -16,19 +16,27 @@ once a repository has been set up with `rad lfs init` (from the fork above). ## How it works -Every peer uses their own local IPFS node — there's no shared server: +Every peer uses their own local IPFS node — there's no shared server. All of the actual +IPFS add/pin work and git-notes bookkeeping (`refs/notes/rad-lfs`) lives in the `rad lfs +store`/`rad lfs fetch` subcommands, implemented in the companion +[**radicle-heartwood-lfs**](https://git.hswro.org/mab122/radicle-heartwood-lfs) fork — that's +also where content gets encrypted for private repositories before it ever reaches IPFS. This +crate deliberately has no dependency on Radicle's identity/crypto stack; it just shells out: -- On `git add`/`git commit`, a pre-commit hook (installed by `rad lfs init`) adds each staged - LFS-tracked file to your local IPFS daemon, pins it, and records the resulting CID as a git - note (`refs/notes/rad-lfs`) attached to the LFS pointer file's own git blob hash. That notes - ref travels with the repository via Radicle's normal replication, alongside everything else. +- On `git add`/`git commit`, a pre-commit hook (installed by `rad lfs init`) runs `rad lfs + store` on each staged LFS-tracked file, which adds it to your local IPFS daemon, pins it, + encrypts it first if the repo is private, and records the resulting CID as a git note. That + notes ref travels with the repository via Radicle's normal replication, alongside everything + else. - `rad-lfs-transfer` implements the other half: when `git lfs push`/`pull` runs, it's invoked per-object over stdin/stdout JSON (the standard Git LFS custom-transfer-agent protocol) and: - - **upload**: confirms the object is already pinned in IPFS (it should be, from the - pre-commit hook) or adds it if the hook was bypassed. - - **download**: reconstructs the expected pointer blob hash from the requested oid/size, - looks up its `cid=` note, and fetches the content from your local IPFS daemon (which will - pull it from the wider IPFS network if it isn't local yet). + - **upload**: shells out to `rad lfs store --oid --size `, which is + authoritative and idempotent — it always ends up added, pinned, and noted correctly, + whether or not the pre-commit hook already did it (e.g. if it was bypassed with + `--no-verify`). + - **download**: shells out to `rad lfs fetch --oid --size --out `, which + looks up the note, fetches from IPFS (pulling from the wider network if not local yet), + decrypts if needed, and writes the plaintext to ``. There is no central server anywhere in this design — see [radicle-heartwood-lfs's `LFS-IPFS.md`](https://git.hswro.org/mab122/radicle-heartwood-lfs/src/branch/rad-lfs-ipfs/LFS-IPFS.md) @@ -41,22 +49,30 @@ On Arch Linux: ```sh # To build and install -sudo pacman -S --needed rust git +sudo pacman -S --needed rust # To actually use it (not needed to build/install — see "Requirements" below) sudo pacman -S --needed kubo ``` -On other distributions, install a Rust toolchain (e.g. via [rustup](https://rustup.rs)), Git, -and — only when you want to use LFS, not to build this — [Kubo](https://docs.ipfs.tech/install/). +On other distributions, install a Rust toolchain (e.g. via [rustup](https://rustup.rs)) — +and, only when you want to use LFS, not to build this — [Kubo](https://docs.ipfs.tech/install/). ## Requirements -- Only needed **at runtime**, not to build or install this binary: a local IPFS (Kubo) daemon - reachable at `http://127.0.0.1:5001` (or wherever `KUBO_API_URL` points). If it's not - running when `git lfs push`/`pull` tries to use it, you'll get a clear error telling you to - start one (`ipfs daemon`) — nothing fails silently, and nothing about building or installing - this binary requires IPFS to be present at all. +None of these are needed to build or install this binary — only at runtime, when `git lfs +push`/`pull` actually invokes it: + +- The `rad` binary (built from + [**radicle-heartwood-lfs**](https://git.hswro.org/mab122/radicle-heartwood-lfs)) on your + `PATH`, with the repository already set up via `rad lfs init`. `rad-lfs-transfer` shells out + to `rad lfs store`/`rad lfs fetch` for every object — it does not talk to git notes or IPFS + directly itself anymore (that's all inside those subcommands now), and it has no dependency + on `git` either. +- A local IPFS (Kubo) daemon reachable at `http://127.0.0.1:5001` (or wherever `KUBO_API_URL` + points) — used both by `rad lfs store`/`fetch` themselves and by this binary's own startup + healthcheck on the `init` event, so a daemon-not-running failure is reported clearly up + front (`ipfs daemon`) rather than confusingly mid-transfer. - [Git LFS](https://git-lfs.com) installed (`git-lfs` on your `PATH`) — `git-lfs` package on Arch (`sudo pacman -S --needed git-lfs`). diff --git a/src/kubo.rs b/src/kubo.rs index 47051f6..4269458 100644 --- a/src/kubo.rs +++ b/src/kubo.rs @@ -1,18 +1,19 @@ use anyhow::{Context, bail}; -use serde::Deserialize; +/// Thin client for the local Kubo (IPFS) daemon's HTTP RPC API. +/// +/// All actual add/pin/cat work for LFS objects has moved to `rad lfs +/// store`/`rad lfs fetch` (see `main.rs`), which run with access to the +/// user's Radicle identity and can decide whether to encrypt content for +/// private repos. The only thing this crate still talks to Kubo about +/// directly is a startup healthcheck, so a daemon-not-running failure is +/// reported clearly up front rather than confusingly mid-transfer. #[derive(Clone)] pub struct Kubo { client: reqwest::Client, api_url: String, } -#[derive(Deserialize)] -struct AddResponse { - #[serde(rename = "Hash")] - hash: String, -} - impl Kubo { pub fn new(api_url: String) -> Self { Self { @@ -21,66 +22,6 @@ impl Kubo { } } - /// Adds bytes to IPFS using Kubo's normal chunked UnixFS add (default - /// params — no raw-leaves / single-block tricks) and returns the CID. - pub async fn add(&self, bytes: Vec) -> anyhow::Result { - let part = reqwest::multipart::Part::bytes(bytes).file_name("blob"); - let form = reqwest::multipart::Form::new().part("file", part); - - let resp = self - .client - .post(format!("{}/api/v0/add", self.api_url)) - .multipart(form) - .send() - .await - .context("failed to reach Kubo daemon for /api/v0/add")?; - - if !resp.status().is_success() { - bail!("kubo add failed: HTTP {}", resp.status()); - } - - let body = resp.text().await?; - let line = body - .lines() - .find(|l| !l.trim().is_empty()) - .context("empty response from kubo add")?; - let parsed: AddResponse = serde_json::from_str(line) - .with_context(|| format!("failed to parse kubo add response: {line}"))?; - Ok(parsed.hash) - } - - /// Explicitly (recursively) pins a CID, decoupled from add-time pinning. - pub async fn pin_add(&self, cid: &str) -> anyhow::Result<()> { - let resp = self - .client - .post(format!("{}/api/v0/pin/add", self.api_url)) - .query(&[("arg", cid), ("recursive", "true")]) - .send() - .await - .context("failed to reach Kubo daemon for /api/v0/pin/add")?; - - if !resp.status().is_success() { - bail!("kubo pin/add failed for {cid}: HTTP {}", resp.status()); - } - Ok(()) - } - - /// Streams the full reassembled file content for a CID via /api/v0/cat. - pub async fn cat(&self, cid: &str) -> anyhow::Result { - let resp = self - .client - .post(format!("{}/api/v0/cat", self.api_url)) - .query(&[("arg", cid)]) - .send() - .await - .context("failed to reach Kubo daemon for /api/v0/cat")?; - - if !resp.status().is_success() { - bail!("kubo cat failed for {cid}: HTTP {}", resp.status()); - } - Ok(resp) - } - /// Verifies the local Kubo daemon is actually reachable, for a clear /// upfront error instead of a confusing failure mid-transfer. pub async fn healthcheck(&self) -> anyhow::Result<()> { diff --git a/src/main.rs b/src/main.rs index b8d443d..328b01b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,8 @@ mod kubo; -mod notes; use serde_json::{Value, json}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::process::Command; use kubo::Kubo; @@ -32,11 +32,11 @@ async fn main() -> anyhow::Result<()> { write_line(&mut stdout, &resp).await?; } "upload" => { - let resp = handle_upload(&kubo, &msg).await; + let resp = handle_upload(&msg).await; write_line(&mut stdout, &resp).await?; } "download" => { - let resp = handle_download(&kubo, &msg).await; + let resp = handle_download(&msg).await; write_line(&mut stdout, &resp).await?; } "terminate" => break, @@ -53,8 +53,8 @@ async fn main() -> anyhow::Result<()> { Ok(()) } -async fn handle_upload(kubo: &Kubo, msg: &Value) -> Value { - match upload_inner(kubo, msg).await { +async fn handle_upload(msg: &Value) -> Value { + match upload_inner(msg).await { Ok(oid) => json!({"event": "complete", "oid": oid}), Err((oid, e)) => json!({ "event": "complete", @@ -64,42 +64,47 @@ async fn handle_upload(kubo: &Kubo, msg: &Value) -> Value { } } -async fn upload_inner(kubo: &Kubo, msg: &Value) -> Result { +/// Delegates entirely to `rad lfs store`, which owns add/pin/note-write +/// (and encrypts first, for private repos) on the Radicle side. It's +/// authoritative and idempotent, so there's no separate "already pinned by +/// the pre-commit hook" fast path to maintain here anymore. +async fn upload_inner(msg: &Value) -> Result { let oid = field_str(msg, "oid").map_err(|e| (String::new(), e))?; let size = field_i64(msg, "size").map_err(|e| (oid.clone(), e))?; let path = field_str(msg, "path").map_err(|e| (oid.clone(), e))?; - let blob_sha = notes::pointer_blob_hash(&oid, size) + let output = Command::new("rad") + .args(["lfs", "store", "--oid", &oid, "--size", &size.to_string(), "--", &path]) + .output() .await - .map_err(|e| (oid.clone(), e))?; + .map_err(|e| { + ( + oid.clone(), + anyhow::Error::from(e).context("failed to spawn `rad lfs store`"), + ) + })?; - // The pre-commit hook should already have added+pinned this and written - // the note. If it did, just confirm the pin (idempotent). If not (e.g. - // the hook was bypassed with `--no-verify`), self-heal by adding it now. - match notes::read_cid(&blob_sha).await { - Ok(Some(cid)) => { - kubo.pin_add(&cid).await.map_err(|e| (oid.clone(), e))?; - } - Ok(None) | Err(_) => { - let bytes = tokio::fs::read(&path).await.map_err(|e| { - ( - oid.clone(), - anyhow::Error::from(e).context(format!("failed to read {path}")), - ) - })?; - let cid = kubo.add(bytes).await.map_err(|e| (oid.clone(), e))?; - kubo.pin_add(&cid).await.map_err(|e| (oid.clone(), e))?; - notes::write_cid(&blob_sha, &cid) - .await - .map_err(|e| (oid.clone(), e))?; - } + if !output.status.success() { + return Err(( + oid, + anyhow::anyhow!("rad lfs store failed: {}", String::from_utf8_lossy(&output.stderr)), + )); + } + + // `rad lfs store` prints only the resulting CID on success. The custom-transfer + // protocol's "complete" response must echo back the requested oid (not the CID) so + // git-lfs can match it to the in-flight transfer, but we still sanity-check that a + // CID actually came back rather than trusting the exit code alone. + let cid = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if cid.is_empty() { + return Err((oid, anyhow::anyhow!("rad lfs store exited successfully but printed no CID"))); } Ok(oid) } -async fn handle_download(kubo: &Kubo, msg: &Value) -> Value { - match download_inner(kubo, msg).await { +async fn handle_download(msg: &Value) -> Value { + match download_inner(msg).await { Ok((oid, path)) => json!({"event": "complete", "oid": oid, "path": path}), Err((oid, e)) => json!({ "event": "complete", @@ -109,44 +114,40 @@ async fn handle_download(kubo: &Kubo, msg: &Value) -> Value { } } -async fn download_inner(kubo: &Kubo, msg: &Value) -> Result<(String, String), (String, anyhow::Error)> { +/// Delegates entirely to `rad lfs fetch`, which looks up the note, fetches +/// from IPFS, and decrypts (if needed) on the Radicle side. +async fn download_inner(msg: &Value) -> Result<(String, String), (String, anyhow::Error)> { let oid = field_str(msg, "oid").map_err(|e| (String::new(), e))?; let size = field_i64(msg, "size").map_err(|e| (oid.clone(), e))?; - let blob_sha = notes::pointer_blob_hash(&oid, size) - .await - .map_err(|e| (oid.clone(), e))?; + let tmp_path = std::env::temp_dir().join(format!("rad-lfs-{oid}")); - let cid = notes::read_cid(&blob_sha) + let output = Command::new("rad") + .args([ + "lfs", + "fetch", + "--oid", + &oid, + "--size", + &size.to_string(), + "--out", + ]) + .arg(&tmp_path) + .output() .await - .map_err(|e| (oid.clone(), e))? - .ok_or_else(|| { + .map_err(|e| { ( oid.clone(), - anyhow::anyhow!( - "no CID recorded for oid {oid} (no rad-lfs note on pointer blob {blob_sha}); \ - the peer that has this object needs to push its refs/notes/rad-lfs ref" - ), + anyhow::Error::from(e).context("failed to spawn `rad lfs fetch`"), ) })?; - let mut resp = kubo.cat(&cid).await.map_err(|e| (oid.clone(), e))?; - - let tmp_path = std::env::temp_dir().join(format!("rad-lfs-{oid}")); - let mut file = tokio::fs::File::create(&tmp_path) - .await - .map_err(|e| (oid.clone(), anyhow::Error::from(e)))?; - - while let Some(chunk) = resp - .chunk() - .await - .map_err(|e| (oid.clone(), anyhow::Error::from(e)))? - { - file.write_all(&chunk) - .await - .map_err(|e| (oid.clone(), anyhow::Error::from(e)))?; + if !output.status.success() { + return Err(( + oid, + anyhow::anyhow!("rad lfs fetch failed: {}", String::from_utf8_lossy(&output.stderr)), + )); } - file.flush().await.map_err(|e| (oid.clone(), anyhow::Error::from(e)))?; Ok((oid, tmp_path.to_string_lossy().to_string())) } diff --git a/src/notes.rs b/src/notes.rs deleted file mode 100644 index 0db4250..0000000 --- a/src/notes.rs +++ /dev/null @@ -1,89 +0,0 @@ -use anyhow::{Context, bail}; -use tokio::process::Command; - -pub const NOTES_REF: &str = "refs/notes/rad-lfs"; - -/// Computes the git blob hash for the standard 3-line LFS pointer text for a -/// given oid/size, without needing the file itself or a repo search — the -/// pointer format is fully deterministic. Shells out to `git hash-object` -/// rather than hand-rolling the hash so this automatically respects the -/// repo's actual object format (SHA-1 vs SHA-256). -pub async fn pointer_blob_hash(oid: &str, size: i64) -> anyhow::Result { - let pointer = pointer_text(oid, size); - - let mut child = Command::new("git") - .args(["hash-object", "--stdin", "-t", "blob"]) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn `git hash-object`")?; - - { - use tokio::io::AsyncWriteExt; - let stdin = child.stdin.as_mut().context("no stdin for git hash-object")?; - stdin.write_all(pointer.as_bytes()).await?; - } - - let output = child.wait_with_output().await?; - if !output.status.success() { - bail!( - "git hash-object failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - Ok(String::from_utf8(output.stdout)?.trim().to_string()) -} - -pub fn pointer_text(oid: &str, size: i64) -> String { - format!("version https://git-lfs.github.com/spec/v1\noid sha256:{oid}\nsize {size}\n") -} - -/// Reads the `cid=` note attached to a blob, if any. -pub async fn read_cid(blob_sha: &str) -> anyhow::Result> { - let output = Command::new("git") - .args(["notes", "--ref", NOTES_REF, "show", blob_sha]) - .output() - .await - .context("failed to spawn `git notes show`")?; - - if !output.status.success() { - // Non-zero exit typically means no note exists for this object. - return Ok(None); - } - - let text = String::from_utf8(output.stdout)?; - Ok(parse_cid(&text)) -} - -/// Writes/overwrites the `cid=` note on a blob. -pub async fn write_cid(blob_sha: &str, cid: &str) -> anyhow::Result<()> { - let output = Command::new("git") - .args([ - "notes", - "--ref", - NOTES_REF, - "add", - "-f", - "-m", - &format!("cid={cid}"), - blob_sha, - ]) - .output() - .await - .context("failed to spawn `git notes add`")?; - - if !output.status.success() { - bail!( - "git notes add failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - Ok(()) -} - -fn parse_cid(note: &str) -> Option { - note.lines() - .find_map(|line| line.strip_prefix("cid=")) - .map(|s| s.trim().to_string()) -}