Reproducible cross-compiled builds
Implement a new build pipeline using `podman` and `zig` that is reproducible and can be run entirely on linux. We also simplify the versioning system, defaulting to the output of `git describe` when there are no exact tag matches.
This commit is contained in:
cloudhead
parent
0b5fa51a92
commit
95b5191533
|
|
@ -1,2 +1,3 @@
|
|||
/target
|
||||
/radicle-cli/target
|
||||
/build/artifacts
|
||||
|
|
|
|||
70
build.rs
70
build.rs
|
|
@ -4,7 +4,8 @@ use std::process::Command;
|
|||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Set a build-time `GIT_HEAD` env var which includes the commit id;
|
||||
// such that we can tell which code is running.
|
||||
let hash = Command::new("git")
|
||||
let hash = env::var("GIT_HEAD").unwrap_or_else(|_| {
|
||||
Command::new("git")
|
||||
.arg("rev-parse")
|
||||
.arg("--short")
|
||||
.arg("HEAD")
|
||||
|
|
@ -17,63 +18,21 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or(env::var("GIT_HEAD").unwrap_or("unknown".into()));
|
||||
.unwrap_or("unknown".into())
|
||||
});
|
||||
|
||||
let tags = Command::new("git")
|
||||
.arg("tag")
|
||||
.arg("--points-at")
|
||||
.arg("HEAD")
|
||||
.output()
|
||||
.ok()
|
||||
.and_then(|output| {
|
||||
if output.status.success() {
|
||||
String::from_utf8(output.stdout).ok()
|
||||
let version = if let Ok(version) = env::var("RADICLE_VERSION") {
|
||||
version
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let tags = tags
|
||||
.split_terminator('\n')
|
||||
.filter_map(|s| s.strip_prefix('v'))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if tags.len() > 1 {
|
||||
return Err("More than one version tag found for commit {hash}: {tags:?}".into());
|
||||
}
|
||||
// Used for `RADICLE_VERSION` env.
|
||||
let version = if let Some(version) = tags.first() {
|
||||
// There's a tag pointing at this `HEAD`.
|
||||
// Eg. "1.0.43".
|
||||
Some((*version).to_owned())
|
||||
} else {
|
||||
// If `HEAD` doesn't have a tag pointing to it, this is a development version,
|
||||
// so find the closest tag starting with `v` and append `-dev` to the version.
|
||||
// Eg. "1.0.43-dev".
|
||||
Command::new("git")
|
||||
.arg("describe")
|
||||
.arg("--match=v*") // Match tags starting with `v`
|
||||
.arg("--candidates=1") // Only one result
|
||||
.arg("--abbrev=0") // Don't add the commit short-hash to the tag name
|
||||
.arg("HEAD")
|
||||
.output()
|
||||
.ok()
|
||||
.and_then(|output| {
|
||||
if output.status.success() {
|
||||
String::from_utf8(output.stdout).ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.map(|last| format!("{}-dev", last.trim()))
|
||||
}
|
||||
// If there are no tags found, we'll just call this a pre-release.
|
||||
.unwrap_or(String::from("pre-release"));
|
||||
"pre-release".to_owned()
|
||||
};
|
||||
|
||||
// Set a build-time `GIT_COMMIT_TIME` env var which includes the commit time.
|
||||
let commit_time = Command::new("git")
|
||||
.arg("show")
|
||||
.arg("--format=%ct")
|
||||
let commit_time = env::var("GIT_COMMIT_TIME").unwrap_or_else(|_| {
|
||||
Command::new("git")
|
||||
.arg("log")
|
||||
.arg("-1")
|
||||
.arg("--pretty=%ct")
|
||||
.arg("HEAD")
|
||||
.output()
|
||||
.ok()
|
||||
|
|
@ -84,7 +43,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or(0.to_string());
|
||||
.unwrap_or(0.to_string())
|
||||
});
|
||||
|
||||
println!("cargo::rustc-env=RADICLE_VERSION={version}");
|
||||
println!("cargo::rustc-env=GIT_COMMIT_TIME={commit_time}");
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
# Builds release binaries for Radicle.
|
||||
FROM rust:1.77.2-alpine3.19 as builder
|
||||
LABEL maintainer="Radicle Team <team@radicle.xyz>"
|
||||
WORKDIR /src
|
||||
COPY . .
|
||||
|
||||
# Copy cargo configuration we're going to use to specify compiler options.
|
||||
RUN mkdir -p .cargo && cp build/config.toml .cargo/config.toml
|
||||
# Install dependencies.
|
||||
RUN apk update && apk add --no-cache mold clang git musl-dev minisign curl xz asciidoctor
|
||||
# Build man pages and strip metadata. Removes all comments, since they include
|
||||
# non-reproducible information, such as version numbers.
|
||||
RUN asciidoctor --doctype manpage --backend manpage --destination-dir . *.1.adoc && \
|
||||
find . -maxdepth 1 -type f -name '*.1' -exec sed -i '/^.\\\"/d' '{}' \;
|
||||
# Add cargo targets.
|
||||
RUN rustup target add \
|
||||
x86_64-unknown-linux-musl \
|
||||
aarch64-unknown-linux-musl \
|
||||
x86_64-apple-darwin \
|
||||
aarch64-apple-darwin
|
||||
|
||||
# Linux x86_64 and aarch64 build.
|
||||
ENV CC=clang C_INCLUDE_PATH=/usr/include/x86_64-linux-musl
|
||||
RUN cargo build --locked --release \
|
||||
--target=x86_64-unknown-linux-musl \
|
||||
--target=aarch64-unknown-linux-musl \
|
||||
-p radicle-node \
|
||||
-p radicle-httpd \
|
||||
-p radicle-remote-helper \
|
||||
-p radicle-cli
|
||||
|
||||
# Install dependencies for cross-compiling to macOS.
|
||||
# We use Zig as the linker to perform the compilation from a Linux host.
|
||||
# Zig is not yet available on Debian, so we download the official binary.
|
||||
# Compilation is done via `cargo-zigbuild` which is a wrapper around `zig`.
|
||||
RUN curl -sSf -o zig.tar.xz https://ziglang.org/builds/zig-linux-x86_64-0.12.0-dev.3678+130fb5cb0.tar.xz && \
|
||||
curl -sSf -o zig.tar.xz.minisig https://ziglang.org/builds/zig-linux-x86_64-0.12.0-dev.3678+130fb5cb0.tar.xz.minisig && \
|
||||
minisign -Vm zig.tar.xz -P RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U && \
|
||||
xz -d -c zig.tar.xz | tar -x && \
|
||||
mv zig-linux-x86_64-0.12.0-dev.3678+130fb5cb0/zig /usr/bin/zig && \
|
||||
mv zig-linux-x86_64-0.12.0-dev.3678+130fb5cb0/lib /usr/lib/zig && \
|
||||
cargo install cargo-zigbuild@0.18.3
|
||||
|
||||
|
||||
# Parts of the macOS SDK are required to build Radicle, we make these available
|
||||
# here. So far only `CoreFoundation` and `Security` frameworks are needed.
|
||||
RUN xz -d -c build/macos-sdk-11.3.tar.xz | tar -x
|
||||
# This env var is used by `cargo-zigbuild` to find the SDK.
|
||||
ENV SDKROOT /src/macos-sdk-11.3
|
||||
|
||||
# Darwin x86_64 and aarch64 build.
|
||||
RUN cargo zigbuild --locked --release \
|
||||
--target=x86_64-apple-darwin \
|
||||
--target=aarch64-apple-darwin \
|
||||
-p radicle-node \
|
||||
-p radicle-httpd \
|
||||
-p radicle-remote-helper \
|
||||
-p radicle-cli
|
||||
|
||||
# Now copy the files to a new image without all the intermediary artifacts to
|
||||
# save some space.
|
||||
FROM alpine:3.19 as packager
|
||||
COPY --from=builder \
|
||||
/src/*.1 \
|
||||
/src/target/x86_64-unknown-linux-musl/release/rad \
|
||||
/src/target/x86_64-unknown-linux-musl/release/git-remote-rad \
|
||||
/src/target/x86_64-unknown-linux-musl/release/radicle-node \
|
||||
/src/target/x86_64-unknown-linux-musl/release/radicle-httpd \
|
||||
/builds/x86_64-unknown-linux-musl/
|
||||
COPY --from=builder \
|
||||
/src/*.1 \
|
||||
/src/target/aarch64-unknown-linux-musl/release/rad \
|
||||
/src/target/aarch64-unknown-linux-musl/release/git-remote-rad \
|
||||
/src/target/aarch64-unknown-linux-musl/release/radicle-node \
|
||||
/src/target/aarch64-unknown-linux-musl/release/radicle-httpd \
|
||||
/builds/aarch64-unknown-linux-musl/
|
||||
COPY --from=builder \
|
||||
/src/*.1 \
|
||||
/src/target/aarch64-apple-darwin/release/rad \
|
||||
/src/target/aarch64-apple-darwin/release/git-remote-rad \
|
||||
/src/target/aarch64-apple-darwin/release/radicle-node \
|
||||
/src/target/aarch64-apple-darwin/release/radicle-httpd \
|
||||
/builds/aarch64-apple-darwin/
|
||||
COPY --from=builder \
|
||||
/src/*.1 \
|
||||
/src/target/x86_64-apple-darwin/release/rad \
|
||||
/src/target/x86_64-apple-darwin/release/git-remote-rad \
|
||||
/src/target/x86_64-apple-darwin/release/radicle-node \
|
||||
/src/target/x86_64-apple-darwin/release/radicle-httpd \
|
||||
/builds/x86_64-apple-darwin/
|
||||
|
||||
# Create and compress reproducible archive.
|
||||
WORKDIR /builds
|
||||
RUN apk update && apk add --no-cache tar xz
|
||||
RUN find * -maxdepth 0 -type d -exec mv '{}' "radicle-$RADICLE_VERSION-{}" \; && \
|
||||
find * -maxdepth 0 -type d -exec tar \
|
||||
--sort=name \
|
||||
--verbose \
|
||||
--mtime="@$GIT_COMMIT_TIME" \
|
||||
--owner=0 \
|
||||
--group=0 \
|
||||
--numeric-owner \
|
||||
--format=posix \
|
||||
--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \
|
||||
--mode='go+u,go-w' \
|
||||
--create --xz \
|
||||
--file="{}.tar.xz" \
|
||||
'{}' \;
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
# Builds
|
||||
|
||||
Radicle uses a [reproducible build][rb] pipeline to make binary verification
|
||||
easier and more secure.
|
||||
|
||||
[rb]: https://reproducible-builds.org/
|
||||
|
||||
This build pipeline is designed to be run on an x86_64 machine running Linux.
|
||||
The output is a set of `.tar.xz` archives containing binaries for the supported
|
||||
platforms and signed by the user's Radicle key.
|
||||
|
||||
These binaries are statically linked to be maximally portable, and designed to
|
||||
be reproducible, byte for byte.
|
||||
|
||||
To run the build, simply enter the following command from the repository root:
|
||||
|
||||
build/build.sh
|
||||
|
||||
This will build all targets and place the output in `build/artifacts` with
|
||||
one sub-directory per build target.
|
||||
|
||||
Note that it will use `git describe` to get a version number for the build.
|
||||
You *must* have a commit tagged with a version in your history or the build
|
||||
will fail, eg. `v1.0.0`.
|
||||
|
||||
When the build completes, the SHA-256 checksums of the artifacts are output.
|
||||
For a given Radicle version and source tree, the same set of checksums should
|
||||
always be output, no matter where or when the build is run. If they do not
|
||||
match, either the build pipeline has a bug, making it non-reproducible, or one
|
||||
of the machines is compromised.
|
||||
|
||||
Here's an example output for a development version of Radicle:
|
||||
|
||||
b9aa75bba175e18e05df4f6b39ec097414bbf56ccdeb4a2229b557f8b8e05404 radicle-1.0.0-rc.4-3-gb299f3b5-aarch64-apple-darwin.tar.xz
|
||||
c7070806bf2d17a8a0d3b329e4d57b1e544b7b82cb58e2863074d96348a2ab0d radicle-1.0.0-rc.4-3-gb299f3b5-aarch64-unknown-linux-musl.tar.xz
|
||||
1a8327854f16ea90491fb90e0c3291a63c4b2ab01742c8435faec7d370cacb79 radicle-1.0.0-rc.4-3-gb299f3b5-x86_64-apple-darwin.tar.xz
|
||||
709ac67541ff0c0c570ac22ab2de9f98320e0cc2cc9b67f1909c014a2bb5bd49 radicle-1.0.0-rc.4-3-gb299f3b5-x86_64-unknown-linux-musl.tar.xz
|
||||
|
||||
A script is included in `build/checksums.sh` to output these checksums after
|
||||
the artifacts are built.
|
||||
|
||||
## Requirements
|
||||
|
||||
The following software is required for the build:
|
||||
|
||||
* `podman`
|
||||
* `rad` (The Radicle CLI)
|
||||
* `sha256sum`
|
||||
|
||||
## macOS
|
||||
|
||||
macOS binaries are not signed or notarized, so they have to be downloaded via
|
||||
the CLI to avoid issues. A copy of a small subset of the Apple SDK is included
|
||||
here to be able to cross-compile.
|
||||
|
||||
## Podman
|
||||
|
||||
We use `podman` to make the build reproducible on any machine by controlling
|
||||
the build environment. We prefer `podman` to `docker` because it doesn't
|
||||
require a background process to run and can be run without root access out of
|
||||
the box.
|
||||
|
||||
The first time you run `podman`, you may have to give yourself some extra UIDs
|
||||
for `podman` to use, with:
|
||||
|
||||
sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $USER
|
||||
|
||||
Then update `podman` with:
|
||||
|
||||
podman system migrate
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
main() {
|
||||
# Use UTC time for everything.
|
||||
export TZ=UTC0
|
||||
# Set minimal locale.
|
||||
export LC_ALL=C
|
||||
# Set source date. This is honored by `asciidoctor` and other tools.
|
||||
export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
|
||||
|
||||
if ! command -v rad > /dev/null; then
|
||||
echo "fatal: rad is not installed" ; exit 1
|
||||
fi
|
||||
|
||||
if ! command -v podman > /dev/null; then
|
||||
echo "fatal: podman is not installed" ; exit 1
|
||||
fi
|
||||
|
||||
if ! command -v sha256sum > /dev/null; then
|
||||
echo "fatal: sha256sum is not installed" ; exit 1
|
||||
fi
|
||||
|
||||
rev="$(git rev-parse --short HEAD)"
|
||||
tempdir="$(mktemp -d)"
|
||||
gitarchive="$tempdir/heartwood-$rev.tar.gz"
|
||||
keypath="$(rad path)/keys/radicle.pub"
|
||||
|
||||
if ! version="$(git describe --match='v*' --candidates=1 2>/dev/null)"; then
|
||||
echo "fatal: no version tag found by 'git describe'" ; exit 1
|
||||
fi
|
||||
# Remove `v` prefix from version.
|
||||
version=${version#v}
|
||||
image=radicle-build-$version
|
||||
|
||||
if [ ! -f "$keypath" ]; then
|
||||
echo "fatal: no key found at $keypath" ; exit 1
|
||||
fi
|
||||
# Authenticate user for signing
|
||||
rad auth
|
||||
|
||||
echo "Building Radicle $version.."
|
||||
echo "Creating archive of repository at $rev in $gitarchive.."
|
||||
git archive --format tar.gz -o "$gitarchive" HEAD
|
||||
|
||||
echo "Building image ($image).."
|
||||
podman --cgroup-manager=cgroupfs build \
|
||||
--env SOURCE_DATE_EPOCH \
|
||||
--env GIT_COMMIT_TIME=$SOURCE_DATE_EPOCH \
|
||||
--env GIT_HEAD=$rev \
|
||||
--env RADICLE_VERSION=$version \
|
||||
--arch amd64 --tag $image -f ./build/Dockerfile - < $gitarchive
|
||||
|
||||
echo "Creating container (radicle-build-container).."
|
||||
podman --cgroup-manager=cgroupfs create --replace --name radicle-build-container $image
|
||||
|
||||
targets="\
|
||||
x86_64-unknown-linux-musl \
|
||||
aarch64-unknown-linux-musl \
|
||||
x86_64-apple-darwin \
|
||||
aarch64-apple-darwin"
|
||||
|
||||
for target in $targets; do
|
||||
outdir=build/artifacts/$target
|
||||
|
||||
echo "Copying artifacts for $target.."
|
||||
mkdir -p $outdir
|
||||
rm -rf $outdir/*
|
||||
|
||||
filename="radicle-$version-$target.tar.xz"
|
||||
filepath="$outdir/$filename"
|
||||
|
||||
# Copy archive to target folder.
|
||||
podman cp radicle-build-container:/builds/$filename $outdir
|
||||
|
||||
# Output SHA256 digest of archive.
|
||||
checksum="$(cd $outdir && sha256sum $filename)"
|
||||
echo "Checksum of $filepath is $(echo "$checksum" | cut -d' ' -f1)"
|
||||
echo "$checksum" > $filepath.sha256
|
||||
|
||||
# Sign archive and verify archive.
|
||||
rm -f $filepath.sig # Delete existing signature
|
||||
ssh-keygen -Y sign -n file -f $keypath $filepath
|
||||
ssh-keygen -Y check-novalidate -n file -s $filepath.sig < $filepath
|
||||
done
|
||||
|
||||
# Remove build artifacts that aren't needed anymore.
|
||||
rm -f $gitarchive
|
||||
podman rm radicle-build-container > /dev/null
|
||||
podman rmi --ignore localhost/$image
|
||||
}
|
||||
|
||||
# Run build.
|
||||
echo "Running build.."
|
||||
main "$@"
|
||||
|
||||
# Show artifact checksums.
|
||||
echo
|
||||
build/checksums.sh
|
||||
echo
|
||||
|
||||
echo "Build ran successfully."
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
#!/bin/sh
|
||||
find build/artifacts -type f -name '*.sha256' -exec cat {} +
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
[target.x86_64-unknown-linux-musl]
|
||||
linker = "clang"
|
||||
rustflags = [
|
||||
"-C", "link-arg=-fuse-ld=mold",
|
||||
"-C", "link-arg=--target=x86_64-unknown-linux-musl",
|
||||
"-C", "codegen-units=1",
|
||||
"-C", "incremental=false",
|
||||
"-C", "opt-level=3",
|
||||
]
|
||||
|
||||
[target.aarch64-unknown-linux-musl]
|
||||
linker = "clang"
|
||||
rustflags = [
|
||||
"-C", "link-arg=-fuse-ld=mold",
|
||||
"-C", "link-arg=--target=aarch64-unknown-linux-musl",
|
||||
"-C", "codegen-units=1",
|
||||
"-C", "incremental=false",
|
||||
"-C", "opt-level=3",
|
||||
]
|
||||
Binary file not shown.
|
|
@ -21,11 +21,8 @@ impl<'a> Version<'a> {
|
|||
..
|
||||
} = self;
|
||||
|
||||
if version.ends_with("-dev") {
|
||||
writeln!(w, "{name} {version}+{commit}")?;
|
||||
} else {
|
||||
writeln!(w, "{name} {version} ({commit})")?;
|
||||
};
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -52,17 +49,5 @@ mod test {
|
|||
.unwrap();
|
||||
let res = std::str::from_utf8(&buffer).unwrap();
|
||||
assert_eq!("rad 1.2.3 (28b341d)\n", res);
|
||||
|
||||
let mut buffer = Vec::new();
|
||||
Version {
|
||||
name: "rad",
|
||||
version: "1.2.3-dev",
|
||||
commit: "28b341d",
|
||||
timestamp: "",
|
||||
}
|
||||
.write(&mut buffer)
|
||||
.unwrap();
|
||||
let res = std::str::from_utf8(&buffer).unwrap();
|
||||
assert_eq!("rad 1.2.3-dev+28b341d\n", res);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,30 @@
|
|||
|
||||
set -e
|
||||
|
||||
# Attempt to install `asciidoctor` on Debian, Arch Linux and MacOS.
|
||||
install() {
|
||||
os="$(uname)"
|
||||
|
||||
case "$os" in
|
||||
Linux)
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
# Debian
|
||||
apt-get update
|
||||
apt-get install -y asciidoctor
|
||||
elif command -v pacman >/dev/null 2>&1; then
|
||||
# Arch Linux
|
||||
pacman -Sy --noconfirm asciidoctor
|
||||
fi ;;
|
||||
Darwin) # MacOS
|
||||
if command -v brew >/dev/null 2>&1; then
|
||||
brew install asciidoctor
|
||||
fi ;;
|
||||
*)
|
||||
echo "fatal: unknown operating system: $os"
|
||||
exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
main() {
|
||||
if [ $# -lt 2 ]; then
|
||||
echo "usage: $0 <output-dir> <input-file>..."
|
||||
|
|
@ -11,6 +35,11 @@ main() {
|
|||
outdir="$1"
|
||||
shift
|
||||
|
||||
if ! command -v asciidoctor >/dev/null 2>&1; then
|
||||
echo "Installing 'asciidoctor'.."
|
||||
install
|
||||
fi
|
||||
|
||||
for input in "$@"; do
|
||||
echo "Building $input.."
|
||||
asciidoctor --doctype manpage --backend manpage --destination-dir "$outdir" "$input"
|
||||
|
|
|
|||
Loading…
Reference in New Issue