ci: promote released sources and Nix packages together

This commit is contained in:
Kevin Yue committed 2026-10-23 11:47:47 +08:00
1 parent 69e4183202
commit 1ea9327f18
12 files changed
+619 -100

No files matched your search

-59
View File
@@ -1,59 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
release_version="${1:-}"
release_tag="${2:-}"
if git diff --quiet -- flake.nix flake.lock; then
echo "flake hashes and lock are already up to date"
exit 0
fi
if [[ -z "$release_version" ]]; then
release_version="$(grep '^version' Cargo.toml | head -1 | sed 's/version *= *"\(.*\)"/\1/')"
fi
version="${release_version#v}"
if [[ -z "$release_tag" ]]; then
release_tag="v$version"
fi
branch_suffix="v${version}"
title="chore: update flake hashes for v${version}"
body="Updates the Nix flake fetchurl hashes for v${version} release assets and refreshes the Nixpkgs lock."
if [[ "$release_tag" != "v$version" ]]; then
branch_suffix="${branch_suffix}-${release_tag}"
title="chore: update flake hashes for v${version} from ${release_tag}"
body="Updates the Nix flake fetchurl hashes for v${version} using ${release_tag} release assets and refreshes the Nixpkgs lock."
fi
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
if [[ "$release_tag" == "snapshot" ]]; then
source_branch="${GITHUB_REF_NAME:-$(git branch --show-current)}"
source_branch_slug="$(printf '%s' "$source_branch" | sed -E 's#[/[:space:]]+#-#g; s#[^A-Za-z0-9._-]##g; s#^-+##; s#-+$##')"
branch="chore/flake-${source_branch_slug}"
git switch -C "$branch"
git add flake.nix flake.lock
git commit -m "$title"
git push --force origin "$branch"
echo "Updated snapshot flake hashes on branch $branch from $source_branch; skipping PR creation"
exit 0
fi
branch="chore/update-flake-hashes-${branch_suffix}"
git switch -C "$branch"
git add flake.nix flake.lock
git commit -m "$title"
git push --force origin "$branch"
pr_number="$(gh pr list --head "$branch" --state open --json number --jq '.[0].number')"
if [[ -n "$pr_number" ]]; then
gh pr edit "$pr_number" --title "$title" --body "$body"
else
gh pr create --title "$title" --body "$body" --base main --head "$branch"
fi
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env bash
# Prepare a reviewable main candidate; publishing and merging remain explicit.
set -euo pipefail
tag="${1:?Usage: prepare-release-promotion.sh <vX.Y.Z> <updated-flake-directory>}"
artifact_dir="$(cd "${2:?Updated flake artifact directory is required}" && pwd)"
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
if [[ ! "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "A stable release tag is required" >&2
exit 1
fi
if [[ -n "$(git status --porcelain)" ]]; then
echo "Start with a clean working tree" >&2
exit 1
fi
python3 - "$artifact_dir/release-candidate.json" "$tag" <<'PY'
import json, sys
from pathlib import Path
candidate = json.loads(Path(sys.argv[1]).read_text())
if candidate['releaseTag'] != sys.argv[2] or candidate['version'] != sys.argv[2][1:]:
raise SystemExit('Flake artifact does not belong to the requested release')
PY
python3 "$script_dir/validate-release-version.py" --root "$artifact_dir" --version "${tag#v}"
git fetch origin refs/heads/main:refs/remotes/origin/main
git fetch origin "refs/tags/$tag"
release_sha="$(git rev-parse 'FETCH_HEAD^{commit}')"
python3 - "$tag" <<'PY'
import re, subprocess, sys
cargo = subprocess.check_output(['git', 'show', 'origin/main:Cargo.toml'], text=True)
version = re.search(r'^version\s*=\s*"([0-9.]+)"', cargo, re.M)[1]
if tuple(map(int, sys.argv[1][1:].split('.'))) < tuple(map(int, version.split('.'))):
raise SystemExit('Refusing to prepare a release older than main')
PY
branch="chore/promote-$tag"
git switch -c "$branch" origin/main
git merge --no-commit --no-ff "$release_sha"
cp "$artifact_dir/flake.nix" "$artifact_dir/flake.lock" .
python3 "$script_dir/validate-release-version.py" --version "${tag#v}"
# Check against Cargo too: overriding the expectation above is only for artifacts.
python3 "$script_dir/validate-release-version.py"
git add flake.nix flake.lock
if ! git diff --cached --quiet || [[ -f "$(git rev-parse --git-path MERGE_HEAD)" ]]; then
git commit -m "release: promote $tag sources and Nix packages together"
fi
echo "Prepared $branch. Review the candidate, push normally, and open one PR against main."
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env bash
# Publish a tested snapshot flake beside its source branch; never open a main PR.
set -euo pipefail
source_branch="${GITHUB_REF_NAME:?Run this workflow from the source branch}"
read -r branch expected version < <(python3 - <<'PYMETA'
import json, re
from pathlib import Path
metadata = json.loads(Path('release-candidate.json').read_text())
if metadata['releaseTag'] != 'snapshot':
raise SystemExit('Only snapshot test branches can be published')
head = metadata['expectedTestHead']
if head and not re.fullmatch('[0-9a-f]{40}', head):
raise SystemExit('Invalid test branch lease')
print(metadata['testBranch'], head or '-', metadata['version'])
PYMETA
)
if [[ "$expected" == - ]]; then
expected=""
fi
git check-ref-format "refs/heads/$branch"
python3 scripts/validate-release-version.py --version "$version" --release-tag snapshot
if git diff --quiet -- flake.nix flake.lock; then
echo "Snapshot flake files are already up to date"
exit 0
fi
# The lease was captured before candidate generation and Nix validation.
# Concurrent or delayed runs cannot replace a newer branch update.
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git switch -c "$branch"
git add flake.nix flake.lock
git commit -m "test: prepare snapshot flake for $source_branch"
git push --force-with-lease="refs/heads/$branch:$expected" origin "HEAD:refs/heads/$branch"
echo "Published $branch from $source_branch for Nix testing"
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Retain one source pair per build run so retries cannot move branch refs."""
import argparse
import json
import os
import re
import subprocess
from pathlib import Path
def record(path, gp, gpgui, output):
if path.exists():
sources = json.loads(path.read_text())
else:
sources = {
name: subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"],
text=True, stderr=subprocess.DEVNULL).strip()
for name, root in (("gp_sha", gp), ("gpgui_sha", gpgui))
}
if set(sources) != {"gp_sha", "gpgui_sha"} or any(
not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{40}", value)
for value in sources.values()
):
raise ValueError("invalid retained build source refs")
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(sources, indent=2) + "\n")
with output.open("a") as stream:
for name, sha in sources.items():
stream.write(f"{name}={sha}\n")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("path", type=Path)
parser.add_argument("--gp", type=Path, required=True)
parser.add_argument("--gpgui", type=Path, required=True)
args = parser.parse_args()
try:
record(args.path, args.gp, args.gpgui, Path(os.environ["GITHUB_OUTPUT"]))
except (OSError, ValueError, subprocess.CalledProcessError):
parser.exit(1, "Could not retain the selected build source refs.\n")
if __name__ == "__main__":
main()
+218
View File
@@ -0,0 +1,218 @@
import importlib.util
import json
import os
from pathlib import Path
import re
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[2]
def load(name):
spec = importlib.util.spec_from_file_location(name, ROOT / 'scripts' / f'{name}.py')
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
versions = load('validate-release-version')
sources = load('record-build-sources')
def git(root, *args):
return subprocess.check_output(['git', '-C', str(root), *args], text=True,
stderr=subprocess.DEVNULL).strip()
def initialize(root):
root.mkdir(parents=True, exist_ok=True)
git(root, 'init', '-b', 'main')
git(root, 'config', 'user.name', 'Release Test')
git(root, 'config', 'user.email', '[email protected]')
def write_version(root, cargo='3.1.0', flake=None, tag=None):
flake = flake or cargo
(root / 'Cargo.toml').write_text(f'[workspace.package]\nversion = "{cargo}"\n')
(root / 'flake.nix').write_text(f'releaseVersion = "{flake}";\nreleaseTag = "{tag or "v" + flake}";\n')
(root / 'flake.lock').write_text('{}\n')
def commit(root, title):
git(root, 'add', '.')
git(root, 'commit', '-m', title)
return git(root, 'rev-parse', 'HEAD')
class ReleaseVersionTests(unittest.TestCase):
def test_main_rejects_old_new_and_snapshot_flakes(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
for selected, tag in [('3.1.0', 'v3.1.0'), ('3.3.0', 'v3.3.0'), ('3.2.0', 'snapshot')]:
write_version(root, '3.2.0', selected, tag)
with self.subTest(selected=selected, tag=tag), self.assertRaises(ValueError):
versions.validate(root)
write_version(root, '3.2.0')
self.assertEqual(versions.validate(root), '3.2.0')
def test_snapshot_candidate_is_allowed_outside_main(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
write_version(root, '3.2.0', tag='snapshot')
self.assertEqual(versions.validate(root, release_tag='snapshot'), '3.2.0')
def test_rollback_of_both_version_fields_is_rejected(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
initialize(root)
write_version(root, '3.3.0')
base = commit(root, 'new main')
write_version(root, '3.2.0')
with self.assertRaisesRegex(ValueError, 'older'):
versions.validate(root, base_ref=base)
def test_actual_package_version_must_match(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
write_version(root, '3.2.0')
package = root / 'result'
(package / 'bin').mkdir(parents=True)
for name in ('gpclient', 'gpservice', 'gpauth', 'gpgui'):
tool = package / 'bin' / name
tool.write_text(f'#!/bin/sh\necho "{name} 3.2.0 (PRIVATE_IDENTITY)"\n')
tool.chmod(0o755)
versions.validate(root, package=package)
(package / 'bin/gpgui').write_text('#!/bin/sh\necho "gpgui 3.1.0 PRIVATE_DETAIL"\n')
with self.assertRaisesRegex(ValueError, 'gpgui'):
versions.validate(root, package=package)
class SourceSelectionTests(unittest.TestCase):
def test_retry_uses_original_sources_after_both_branches_advance(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
paths = [root / 'gp', root / 'gpgui']
for repo in paths:
initialize(repo)
(repo / 'source').write_text('first')
commit(repo, 'first')
path, output = root / 'sources.json', root / 'output'
sources.record(path, *paths, output)
original = json.loads(path.read_text())
for repo in paths:
(repo / 'source').write_text('second')
commit(repo, 'second')
sources.record(path, *paths, output)
self.assertEqual(json.loads(path.read_text()), original)
self.assertEqual(output.read_text().count(f"gp_sha={original['gp_sha']}"), 2)
def test_corrupt_retained_sources_do_not_resolve_again(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
path = root / 'sources.json'
path.write_text('{"gp_sha":"main","gpgui_sha":"main"}')
with self.assertRaises(ValueError):
sources.record(path, root, root, root / 'output')
class PromotionTests(unittest.TestCase):
def test_sources_and_flake_are_committed_together_preserving_main_work(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
repo, origin, artifact = root / 'repo', root / 'origin.git', root / 'artifact'
initialize(repo)
write_version(repo)
old = commit(repo, '3.1')
git(repo, 'switch', '-c', 'release/next')
write_version(repo, '3.2.0', '3.1.0')
(repo / 'feature').write_text('released product feature')
release = commit(repo, '3.2 source')
git(repo, 'tag', 'v3.2.0')
git(repo, 'switch', 'main')
(repo / 'infrastructure').write_text('preserve newer main work')
main = commit(repo, 'main infrastructure')
git(root, 'init', '--bare', str(origin))
git(repo, 'remote', 'add', 'origin', str(origin))
git(repo, 'push', 'origin', 'main', 'v3.2.0')
artifact.mkdir()
write_version(artifact, '3.2.0')
(artifact / 'release-candidate.json').write_text(json.dumps({'version': '3.2.0', 'releaseTag': 'v3.2.0'}))
result = subprocess.run(['bash', str(ROOT / 'scripts/prepare-release-promotion.sh'),
'v3.2.0', str(artifact)], cwd=repo, capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(versions.validate(repo), '3.2.0')
self.assertEqual(git(repo, 'rev-parse', 'main'), main)
self.assertEqual(git(repo, 'rev-parse', 'v3.2.0'), release)
self.assertEqual(git(repo, 'show', f'{old}:Cargo.toml'), '[workspace.package]\nversion = "3.1.0"')
self.assertTrue((repo / 'infrastructure').exists())
self.assertTrue((repo / 'feature').exists())
self.assertEqual(git(repo, 'rev-list', '--parents', '-1', 'HEAD').split()[1:], [main, release])
self.assertEqual(git(repo, 'status', '--porcelain'), '')
class SnapshotBranchTests(unittest.TestCase):
def run_publish(self, root, expected, version='3.2.0'):
write_version(root, version, tag='snapshot')
metadata = {'releaseTag': 'snapshot', 'version': version,
'testBranch': 'chore/flake-feature-example', 'expectedTestHead': expected}
(root / 'release-candidate.json').write_text(json.dumps(metadata))
env = dict(os.environ, GITHUB_REF_NAME='feature/example')
# CI checks out the same helper at its selected SHA. Use the real validator here.
(root / 'scripts').mkdir(exist_ok=True)
validator = root / 'scripts/validate-release-version.py'
validator.write_bytes((ROOT / 'scripts/validate-release-version.py').read_bytes())
return subprocess.run(['bash', str(ROOT / 'scripts/publish-flake-test-branch.sh')],
cwd=root, env=env, capture_output=True, text=True)
def test_branch_publication_and_stale_lease(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
repo, origin = root / 'repo', root / 'origin.git'
initialize(repo)
write_version(repo, '3.2.0')
base = commit(repo, 'source')
git(root, 'init', '--bare', str(origin))
git(repo, 'remote', 'add', 'origin', str(origin))
first = self.run_publish(repo, '')
self.assertEqual(first.returncode, 0, first.stdout + first.stderr)
head = git(repo, 'rev-parse', 'HEAD')
self.assertEqual(git(origin, 'rev-parse', 'refs/heads/chore/flake-feature-example'), head)
git(repo, 'switch', '--detach', base)
git(repo, 'branch', '-D', 'chore/flake-feature-example')
# Another run already created the branch after this run captured an absent head.
stale = self.run_publish(repo, '', '3.2.1')
self.assertNotEqual(stale.returncode, 0)
self.assertIn('stale info', stale.stderr)
self.assertEqual(git(origin, 'rev-parse', 'refs/heads/chore/flake-feature-example'), head)
git(repo, 'switch', '--detach', base)
git(repo, 'branch', '-D', 'chore/flake-feature-example')
fresh = self.run_publish(repo, head, '3.2.1')
self.assertEqual(fresh.returncode, 0, fresh.stdout + fresh.stderr)
self.assertNotEqual(git(origin, 'rev-parse', 'refs/heads/chore/flake-feature-example'), head)
self.assertEqual(git(origin, 'branch', '--list', 'main'), '')
class DockerMatrixTests(unittest.TestCase):
def test_workflow_matrix_follows_all_architecture_choices(self):
source = (ROOT / '.github/workflows/release.yaml').read_text()
block = source.split(' const inputs = ${{ toJson(inputs) }}\n', 1)[1].split('\n build:', 1)[0]
# Execute the actual github-script block, including both returned matrices.
lines = []
for line in block.splitlines():
if line.strip() and not line.startswith(' '):
break
lines.append(line[10:])
js = '\n'.join(lines)
for arch, expected in [('all', ['linux/amd64', 'linux/arm64']),
('x86_64', ['linux/amd64']), ('arm64', ['linux/arm64'])]:
harness = 'const core={setOutput:(k,v)=>console.log(v)}; const inputs=' + json.dumps({'arch': arch}) + ';\n'
result = subprocess.check_output(['node', '-e', harness + '(function(){\n' + js + '\n})();'], text=True)
matrix = json.loads(result.splitlines()[0])
self.assertEqual([row['platform'] for row in matrix['include']], expected)
if __name__ == '__main__':
unittest.main()
+70
View File
@@ -0,0 +1,70 @@
#!/usr/bin/env python3
"""Check the release selected by a flake and, optionally, its installed tools."""
import argparse
import re
import subprocess
from pathlib import Path
def cargo_version(root, cargo=None):
if cargo is None:
cargo = (root / "Cargo.toml").read_text()
workspace = re.search(r"(?ms)^\[workspace\.package\]\s*\n(.*?)(?=^\[|\Z)", cargo)
if workspace is None:
raise ValueError("Cargo.toml has no workspace.package section")
return field(workspace[1], "version")
def field(text, name):
matches = re.findall(rf'^\s*{name}\s*=\s*"([^"]+)"\s*;?\s*$', text, re.M)
if len(matches) != 1:
raise ValueError(f"expected one {name} declaration")
return matches[0]
def validate(root, version=None, release_tag=None, package=None, base_ref=None):
expected = version or cargo_version(root)
if not re.fullmatch(r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)", expected):
raise ValueError("expected a stable X.Y.Z version")
tag = release_tag or f"v{expected}"
if tag not in (f"v{expected}", "snapshot"):
raise ValueError("release tag does not match the expected version")
flake = (root / "flake.nix").read_text()
if field(flake, "releaseVersion") != expected or field(flake, "releaseTag") != tag:
raise ValueError(f"flake must select {tag} at version {expected}")
if base_ref:
cargo = subprocess.check_output(["git", "-C", str(root), "show", f"{base_ref}:Cargo.toml"],
text=True, stderr=subprocess.DEVNULL)
base = cargo_version(root, cargo)
if tuple(map(int, expected.split("."))) < tuple(map(int, base.split("."))):
raise ValueError("candidate release is older than the main base")
if package:
for name in ("gpclient", "gpservice", "gpauth", "gpgui"):
result = subprocess.run([str(package / "bin" / name), "--version"],
capture_output=True, text=True, check=True)
# Keep private source identity and other command output out of logs.
match = re.match(rf"{name}\s+(\d+\.\d+\.\d+)(?:\s|$)", result.stdout)
if match is None or match[1] != expected:
raise ValueError(f"installed {name} does not report version {expected}")
return expected
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=Path, default=Path("."))
parser.add_argument("--version", help="Explicit candidate version; defaults to Cargo.toml")
parser.add_argument("--release-tag", help="Explicit candidate tag; defaults to v<version>")
parser.add_argument("--package", type=Path)
parser.add_argument("--base-ref", help="Reject a release older than this main commit")
args = parser.parse_args()
try:
version = validate(args.root, args.version, args.release_tag, args.package, args.base_ref)
except (OSError, ValueError, subprocess.CalledProcessError) as error:
# CalledProcessError does not include captured stdout/stderr in its string.
parser.exit(1, f"Release version check failed: {error}\n")
print(f"Verified release version {version}")
if __name__ == "__main__":
main()