From 1ea9327f18064a0504c8a738d29b27c296854f8d Mon Sep 17 00:00:00 2001 From: Kevin Yue Date: Fri, 23 Oct 2026 11:47:47 +0800 Subject: [PATCH] ci: promote released sources and Nix packages together --- .github/workflows/build.yaml | 105 +++++++--- .github/workflows/nix.yaml | 10 + .github/workflows/release-checks.yaml | 15 ++ .github/workflows/release-version.yaml | 23 +++ .github/workflows/release.yaml | 27 ++- .github/workflows/update-flake-hashes.yaml | 62 +++++- scripts/open-flake-hashes-pr.sh | 59 ------ scripts/prepare-release-promotion.sh | 48 +++++ scripts/publish-flake-test-branch.sh | 36 ++++ scripts/record-build-sources.py | 46 +++++ scripts/tests/test_release_flow.py | 218 +++++++++++++++++++++ scripts/validate-release-version.py | 70 +++++++ 12 files changed, 619 insertions(+), 100 deletions(-) create mode 100644 .github/workflows/release-version.yaml delete mode 100755 scripts/open-flake-hashes-pr.sh create mode 100755 scripts/prepare-release-promotion.sh create mode 100755 scripts/publish-flake-test-branch.sh create mode 100644 scripts/record-build-sources.py create mode 100644 scripts/tests/test_release_flow.py create mode 100644 scripts/validate-release-version.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f23618e..1cfaf9c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,11 +16,11 @@ on: gp_branch: description: 'GlobalProtect-openconnect ref to build (use the matching tag for stable builds)' required: true - default: dev + default: release/next gpgui_branch: description: 'gpgui branch or tag to build and test (use the matching tag for stable builds)' required: true - default: dev + default: release/next build_macos: description: 'Build Apple Silicon artifacts for this run' required: true @@ -54,6 +54,12 @@ concurrency: jobs: validate-inputs: runs-on: ubuntu-latest + permissions: + actions: read + contents: read + outputs: + gp_sha: ${{ steps.sources.outputs.gp_sha }} + gpgui_sha: ${{ steps.sources.outputs.gpgui_sha }} steps: - name: Validate stable refs env: @@ -69,6 +75,53 @@ jobs: fi fi + - name: Checkout CI helpers + uses: actions/checkout@v6 + - name: Restore selected sources on retry + id: restored + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + artifacts="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100")" + existing="$(jq -r '.artifacts[] | select(.name == "build-source-refs") | .id' <<<"$artifacts")" + if [[ -n "$existing" ]]; then + gh run download "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --name build-source-refs --dir selected-sources + echo "cached=true" >> "$GITHUB_OUTPUT" + elif [[ "$GITHUB_RUN_ATTEMPT" -gt 1 ]]; then + echo "Retained source refs are unavailable; start a fresh dispatch." >&2 + exit 1 + else + echo "cached=false" >> "$GITHUB_OUTPUT" + fi + - name: Select public source + if: steps.restored.outputs.cached != 'true' + uses: actions/checkout@v6 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.sha }} + path: selected/gp + - name: Select private source + if: steps.restored.outputs.cached != 'true' + uses: ./.github/actions/checkout-private + with: + token: ${{ secrets.GH_PAT }} + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gpgui_branch || github.ref_name }} + path: selected/gpgui + - name: Record selected sources + id: sources + run: | + python3 scripts/record-build-sources.py selected-sources/sources.json \ + --gp selected/gp --gpgui selected/gpgui + - name: Retain selected sources for retries + if: steps.restored.outputs.cached != 'true' + uses: actions/upload-artifact@v7 + with: + name: build-source-refs + path: selected-sources/sources.json + retention-days: 30 + + test-gp: needs: validate-inputs runs-on: ubuntu-latest @@ -78,7 +131,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} repository: yuezk/GlobalProtect-openconnect - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref }} + ref: ${{ needs.validate-inputs.outputs.gp_sha }} submodules: recursive - name: Run Rust tests as a non-root user with focused privilege checks run: | @@ -118,7 +171,7 @@ jobs: uses: actions/checkout@v6 with: repository: yuezk/GlobalProtect-openconnect - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref }} + ref: ${{ needs.validate-inputs.outputs.gp_sha }} submodules: recursive path: source/gp persist-credentials: false @@ -131,7 +184,7 @@ jobs: uses: ./.ci/.github/actions/checkout-private with: token: ${{ secrets.GH_PAT }} - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gpgui_branch || github.ref_name }} + ref: ${{ needs.validate-inputs.outputs.gpgui_sha }} path: source/gpgui - name: Run frontend and Rust unit tests as an ordinary user timeout-minutes: 80 @@ -152,7 +205,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} repository: yuezk/GlobalProtect-openconnect - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref }} + ref: ${{ needs.validate-inputs.outputs.gp_sha }} path: source/gp submodules: recursive @@ -282,6 +335,7 @@ jobs: build-gp-${{ matrix.package }}/artifacts/* build-docker-image: + needs: validate-inputs if: ${{ !(startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && inputs.channel == 'stable')) }} strategy: fail-fast: false @@ -302,7 +356,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} repository: yuezk/GlobalProtect-openconnect - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref }} + ref: ${{ needs.validate-inputs.outputs.gp_sha }} submodules: recursive - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -364,8 +418,8 @@ jobs: name: build-gpgui (glibc) uses: ./.github/workflows/build-gpgui-linux.yaml with: - gp_ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref }} - gpgui_ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gpgui_branch || github.ref_name }} + gp_ref: ${{ needs.validate-inputs.outputs.gp_sha }} + gpgui_ref: ${{ needs.validate-inputs.outputs.gpgui_sha }} channel: ${{ github.event_name == 'workflow_dispatch' && inputs.channel || 'stable' }} builder: gpgui-builder-tauri2 verify_desktop_assets: true @@ -377,8 +431,8 @@ jobs: name: build-gpgui (musl) uses: ./.github/workflows/build-gpgui-linux.yaml with: - gp_ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref }} - gpgui_ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gpgui_branch || github.ref_name }} + gp_ref: ${{ needs.validate-inputs.outputs.gp_sha }} + gpgui_ref: ${{ needs.validate-inputs.outputs.gpgui_sha }} channel: ${{ github.event_name == 'workflow_dispatch' && inputs.channel || 'stable' }} builder: gpgui-alpine-builder-tauri2 artifact_suffix: -musl @@ -386,7 +440,7 @@ jobs: GH_PAT: ${{ secrets.GH_PAT }} nix-source: - needs: [build-gpgui-glibc] + needs: [validate-inputs, build-gpgui-glibc] strategy: fail-fast: false matrix: @@ -407,7 +461,7 @@ jobs: uses: actions/checkout@v6 with: token: ${{ secrets.GH_PAT }} - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref }} + ref: ${{ needs.validate-inputs.outputs.gp_sha }} submodules: recursive path: gp - name: Download current GUI artifact @@ -435,7 +489,7 @@ jobs: --extra-experimental-features "nix-command flakes" --print-build-logs nix-prebuilt: - needs: [build-gp, build-gpgui-glibc] + needs: [validate-inputs, build-gp, build-gpgui-glibc] strategy: fail-fast: false matrix: @@ -457,7 +511,7 @@ jobs: uses: actions/checkout@v6 with: token: ${{ secrets.GH_PAT }} - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref }} + ref: ${{ needs.validate-inputs.outputs.gp_sha }} submodules: recursive path: gp - name: Download current Linux binary artifact @@ -499,7 +553,7 @@ jobs: build-macos: name: build-macos - needs: [test-gp, test-gpgui] + needs: [validate-inputs, test-gp, test-gpgui] if: ${{ (github.event_name == 'workflow_dispatch' && inputs.build_macos) || (startsWith(github.ref, 'refs/tags/') && vars.BUILD_MACOS_ON_TAG == 'true') }} permissions: contents: read @@ -535,14 +589,14 @@ jobs: uses: ./.ci/.github/actions/checkout-private with: token: ${{ secrets.GH_PAT }} - ref: ${{ (github.event_name == 'workflow_dispatch' && inputs.gpgui_branch || github.ref_name) }} + ref: ${{ needs.validate-inputs.outputs.gpgui_sha }} path: source/gpgui - name: Checkout GlobalProtect-openconnect source uses: actions/checkout@v6 with: repository: yuezk/GlobalProtect-openconnect - ref: ${{ (github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref) }} + ref: ${{ needs.validate-inputs.outputs.gp_sha }} path: source/gp submodules: recursive @@ -815,7 +869,7 @@ jobs: PY build-gpgui-frontend: - needs: setup-bsd-matrix + needs: [validate-inputs, setup-bsd-matrix] if: ${{ needs.setup-bsd-matrix.outputs.enabled == 'true' }} runs-on: ubuntu-latest steps: @@ -829,7 +883,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} repository: yuezk/GlobalProtect-openconnect - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref }} + ref: ${{ needs.validate-inputs.outputs.gp_sha }} path: gpgui-frontend/gp submodules: recursive - name: Checkout CI helpers @@ -841,7 +895,7 @@ jobs: uses: ./.ci/.github/actions/checkout-private with: token: ${{ secrets.GH_PAT }} - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gpgui_branch || github.ref_name }} + ref: ${{ needs.validate-inputs.outputs.gpgui_sha }} path: gpgui-frontend/gpgui - name: Build gpgui frontend run: | @@ -863,6 +917,7 @@ jobs: build-bsd: needs: + - validate-inputs - setup-bsd-matrix - build-gpgui-frontend if: ${{ needs.setup-bsd-matrix.outputs.enabled == 'true' }} @@ -879,7 +934,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} repository: yuezk/GlobalProtect-openconnect - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref }} + ref: ${{ needs.validate-inputs.outputs.gp_sha }} path: ${{ matrix.workspace }}/gp submodules: recursive - name: Checkout CI helpers @@ -891,7 +946,7 @@ jobs: uses: ./.ci/.github/actions/checkout-private with: token: ${{ secrets.GH_PAT }} - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gpgui_branch || github.ref_name }} + ref: ${{ needs.validate-inputs.outputs.gpgui_sha }} path: ${{ matrix.workspace }}/gpgui - name: Download gpgui frontend uses: actions/download-artifact@v8 @@ -1029,6 +1084,7 @@ jobs: attestations: write artifact-metadata: write needs: + - validate-inputs - test-gp - test-gpgui - nix-source @@ -1086,7 +1142,7 @@ jobs: with: token: ${{ secrets.GH_PAT }} repository: yuezk/GlobalProtect-openconnect - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.gp_branch || github.ref }} + ref: ${{ needs.validate-inputs.outputs.gp_sha }} path: gh-release/gp submodules: recursive @@ -1136,6 +1192,7 @@ jobs: publish-docker-image: needs: + - validate-inputs - test-gp - test-gpgui - build-docker-image diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml index 06d28b6..388dffd 100644 --- a/.github/workflows/nix.yaml +++ b/.github/workflows/nix.yaml @@ -3,6 +3,8 @@ name: Nix on: pull_request: paths: + - "Cargo.toml" + - "scripts/validate-release-version.py" - "flake.nix" - "flake.lock" - "nix/**" @@ -11,6 +13,8 @@ on: branches: - main paths: + - "Cargo.toml" + - "scripts/validate-release-version.py" - "flake.nix" - "flake.lock" - "nix/**" @@ -44,6 +48,9 @@ jobs: --extra-experimental-features "nix-command flakes" \ --print-build-logs + - name: Check installed release versions + run: python3 scripts/validate-release-version.py --package result + - name: Inspect prebuilt package run: | test -x result/bin/gpclient @@ -109,6 +116,9 @@ jobs: package ' + - name: Check installed release versions + run: python3 scripts/validate-release-version.py --package result + - name: Inspect module package run: | test -x result/bin/gpclient diff --git a/.github/workflows/release-checks.yaml b/.github/workflows/release-checks.yaml index cb6b2ff..6f8f599 100644 --- a/.github/workflows/release-checks.yaml +++ b/.github/workflows/release-checks.yaml @@ -5,6 +5,11 @@ on: paths: - '.github/workflows/**' - '.github/actions/**' + - 'scripts/record-build-sources.py' + - 'scripts/validate-release-version.py' + - 'scripts/publish-flake-test-branch.sh' + - 'scripts/prepare-release-promotion.sh' + - 'scripts/tests/test_release_flow.py' - 'scripts/select-macos-artifact.sh' - 'scripts/test-private-desktop.sh' - 'scripts/tests/test_private_ci.py' @@ -21,6 +26,11 @@ on: paths: - '.github/workflows/**' - '.github/actions/**' + - 'scripts/record-build-sources.py' + - 'scripts/validate-release-version.py' + - 'scripts/publish-flake-test-branch.sh' + - 'scripts/prepare-release-promotion.sh' + - 'scripts/tests/test_release_flow.py' - 'scripts/select-macos-artifact.sh' - 'scripts/test-private-desktop.sh' - 'scripts/tests/test_private_ci.py' @@ -60,3 +70,8 @@ jobs: shellcheck scripts/select-macos-artifact.sh scripts/test-private-desktop.sh bash -n scripts/test-private-desktop.sh python3 -m unittest discover -s scripts/tests -p 'test_private_ci.py' + - name: Test release promotion and source selection + run: | + bash -n scripts/prepare-release-promotion.sh scripts/publish-flake-test-branch.sh + shellcheck scripts/prepare-release-promotion.sh scripts/publish-flake-test-branch.sh + python3 -m unittest discover -s scripts/tests -p 'test_release_flow.py' diff --git a/.github/workflows/release-version.yaml b/.github/workflows/release-version.yaml new file mode 100644 index 0000000..03e71b2 --- /dev/null +++ b/.github/workflows/release-version.yaml @@ -0,0 +1,23 @@ +name: Release version consistency + +on: + pull_request: + branches: [main] + push: + branches: [main] + +permissions: + contents: read + +jobs: + release-version: + runs-on: ubuntu-latest + steps: + # The default PR checkout is the candidate merged with the current base. + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Check main release version + env: + BASE_REF: ${{ github.event.pull_request.base.sha || github.event.before }} + run: python3 scripts/validate-release-version.py --base-ref "$BASE_REF" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b8af008..07e90cd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -73,6 +73,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.result }} + docker_matrix: ${{ steps.set-matrix.outputs.docker_matrix }} steps: - name: Set up matrix id: set-matrix @@ -88,6 +89,12 @@ jobs: "arm64": ["ubuntu-24.04-arm"] } + const docker = [ + { runner: 'ubuntu-latest', platform: 'linux/amd64' }, + { runner: 'ubuntu-24.04-arm', platform: 'linux/arm64' }, + ].filter(item => osMap[arch].includes(item.runner)) + core.setOutput('docker_matrix', JSON.stringify({ include: docker })) + const package = Object.entries(inputs) .filter(([key, value]) => key.startsWith('release-') && key !== 'release-docker' && value) .map(([key, value]) => key.replace('release-', '')) @@ -145,15 +152,11 @@ jobs: build-docker-image: needs: - check + - setup-matrix if: ${{ inputs.release-docker }} strategy: fail-fast: false - matrix: - include: - - runner: ubuntu-latest - platform: linux/amd64 - - runner: ubuntu-24.04-arm - platform: linux/arm64 + matrix: ${{ fromJson(needs.setup-matrix.outputs.docker_matrix) }} runs-on: ${{ matrix.runner }} steps: - name: Prepare @@ -235,6 +238,17 @@ jobs: IMAGE: yuezk/globalprotect-openconnect TAG: ${{ inputs.tag }} run: | + # Reject an incomplete or unexpected selection before writing any tag. + shopt -s nullglob + digest_files=(*) + expected_count=1 + if [[ "${{ inputs.arch }}" == all ]]; then + expected_count=2 + fi + [[ ${#digest_files[@]} -eq $expected_count ]] + for digest in "${digest_files[@]}"; do + [[ "$digest" =~ ^[0-9a-f]{64}$ ]] + done VERSION="${TAG#v}" tags=(-t "$IMAGE:$TAG") @@ -335,7 +349,6 @@ jobs: if: ${{ inputs.release-binary && inputs.arch == 'all' && needs.gh-release.outputs.stable_version != '' }} permissions: contents: write - pull-requests: write uses: ./.github/workflows/update-flake-hashes.yaml with: version: ${{ needs.gh-release.outputs.stable_version }} diff --git a/.github/workflows/update-flake-hashes.yaml b/.github/workflows/update-flake-hashes.yaml index b6bbcb1..d082f22 100644 --- a/.github/workflows/update-flake-hashes.yaml +++ b/.github/workflows/update-flake-hashes.yaml @@ -20,14 +20,47 @@ on: permissions: contents: write - pull-requests: write jobs: update-flake-hashes: runs-on: ubuntu-latest + outputs: + version: ${{ steps.release.outputs.version }} + release_tag: ${{ steps.release.outputs.release_tag }} steps: - name: Checkout GlobalProtect-openconnect uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + + - name: Resolve candidate release + id: release + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + python3 - <<'PYRELEASE' + import json, os, re, runpy, subprocess + from pathlib import Path + helpers = runpy.run_path('scripts/validate-release-version.py') + version = os.environ['RELEASE_VERSION'].removeprefix('v') or helpers['cargo_version'](Path('.')) + tag = os.environ['RELEASE_TAG'] or f'v{version}' + if not re.fullmatch(r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)', version) or tag not in (f'v{version}', 'snapshot'): + raise SystemExit('Expected a stable version and its matching release tag, or snapshot') + source_slug = re.sub(r'[^A-Za-z0-9._-]', '', re.sub(r'[/\s]+', '-', os.environ['GITHUB_REF_NAME'])).strip('-') + test_branch = f'chore/flake-{source_slug}' + expected_head = '' + if tag == 'snapshot': + remote = subprocess.check_output(['git', 'ls-remote', '--heads', 'origin', f'refs/heads/{test_branch}'], text=True) + expected_head = remote.split()[0] if remote.strip() else '' + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f'version={version}\nrelease_tag={tag}\n') + Path('release-candidate.json').write_text(json.dumps({ + 'version': version, 'releaseTag': tag, 'sourceSha': os.environ['GITHUB_SHA'], + 'workflowRunId': os.environ['GITHUB_RUN_ID'], + 'testBranch': test_branch, 'expectedTestHead': expected_head, + }, indent=2) + '\n') + PYRELEASE - name: Install Nix uses: cachix/install-nix-action@v31 @@ -39,8 +72,8 @@ jobs: - name: Update flake hashes env: - RELEASE_VERSION: ${{ inputs.version }} - RELEASE_TAG: ${{ inputs.release_tag }} + RELEASE_VERSION: ${{ steps.release.outputs.version }} + RELEASE_TAG: ${{ steps.release.outputs.release_tag }} run: | if [[ -n "$RELEASE_VERSION" ]]; then scripts/update-flake-hashes.sh "$RELEASE_VERSION" "$RELEASE_TAG" @@ -55,6 +88,7 @@ jobs: path: | flake.nix flake.lock + release-candidate.json if-no-files-found: error validate-nix-prebuilt: @@ -79,6 +113,8 @@ jobs: steps: - name: Checkout GlobalProtect-openconnect uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} - name: Download updated flake uses: actions/download-artifact@v8 @@ -104,6 +140,12 @@ jobs: --print-build-logs fi + - name: Validate installed release versions + run: | + python3 scripts/validate-release-version.py \ + --version '${{ needs.update-flake-hashes.outputs.version }}' \ + --release-tag '${{ needs.update-flake-hashes.outputs.release_tag }}' --package result + - name: Inspect prebuilt Nix package run: | test -x result/bin/gpclient @@ -126,13 +168,15 @@ jobs: - name: Verify desktop authorization runs on the host run: bash scripts/tests/nix-desktop-authorization.sh result - open-pull-request: - needs: - - validate-nix-prebuilt + publish-test-branch: + if: needs.update-flake-hashes.outputs.release_tag == 'snapshot' + needs: [update-flake-hashes, validate-nix-prebuilt] runs-on: ubuntu-latest steps: - name: Checkout GlobalProtect-openconnect uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} - name: Download updated flake uses: actions/download-artifact@v8 @@ -140,9 +184,7 @@ jobs: name: updated-flake path: . - - name: Open pull request + - name: Publish branch for snapshot testing env: GH_TOKEN: ${{ github.token }} - RELEASE_VERSION: ${{ inputs.version }} - RELEASE_TAG: ${{ inputs.release_tag }} - run: scripts/open-flake-hashes-pr.sh "$RELEASE_VERSION" "$RELEASE_TAG" + run: scripts/publish-flake-test-branch.sh diff --git a/scripts/open-flake-hashes-pr.sh b/scripts/open-flake-hashes-pr.sh deleted file mode 100755 index 4463c83..0000000 --- a/scripts/open-flake-hashes-pr.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -release_version="${1:-}" -release_tag="${2:-}" - -if git diff --quiet -- flake.nix flake.lock; then - echo "flake hashes and lock are already up to date" - exit 0 -fi - -if [[ -z "$release_version" ]]; then - release_version="$(grep '^version' Cargo.toml | head -1 | sed 's/version *= *"\(.*\)"/\1/')" -fi - -version="${release_version#v}" -if [[ -z "$release_tag" ]]; then - release_tag="v$version" -fi - -branch_suffix="v${version}" -title="chore: update flake hashes for v${version}" -body="Updates the Nix flake fetchurl hashes for v${version} release assets and refreshes the Nixpkgs lock." - -if [[ "$release_tag" != "v$version" ]]; then - branch_suffix="${branch_suffix}-${release_tag}" - title="chore: update flake hashes for v${version} from ${release_tag}" - body="Updates the Nix flake fetchurl hashes for v${version} using ${release_tag} release assets and refreshes the Nixpkgs lock." -fi - -git config user.name "github-actions[bot]" -git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - -if [[ "$release_tag" == "snapshot" ]]; then - source_branch="${GITHUB_REF_NAME:-$(git branch --show-current)}" - source_branch_slug="$(printf '%s' "$source_branch" | sed -E 's#[/[:space:]]+#-#g; s#[^A-Za-z0-9._-]##g; s#^-+##; s#-+$##')" - branch="chore/flake-${source_branch_slug}" - git switch -C "$branch" - git add flake.nix flake.lock - git commit -m "$title" - git push --force origin "$branch" - echo "Updated snapshot flake hashes on branch $branch from $source_branch; skipping PR creation" - exit 0 -fi - -branch="chore/update-flake-hashes-${branch_suffix}" - -git switch -C "$branch" -git add flake.nix flake.lock -git commit -m "$title" -git push --force origin "$branch" - -pr_number="$(gh pr list --head "$branch" --state open --json number --jq '.[0].number')" -if [[ -n "$pr_number" ]]; then - gh pr edit "$pr_number" --title "$title" --body "$body" -else - gh pr create --title "$title" --body "$body" --base main --head "$branch" -fi diff --git a/scripts/prepare-release-promotion.sh b/scripts/prepare-release-promotion.sh new file mode 100755 index 0000000..488cb0c --- /dev/null +++ b/scripts/prepare-release-promotion.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Prepare a reviewable main candidate; publishing and merging remain explicit. +set -euo pipefail + +tag="${1:?Usage: prepare-release-promotion.sh }" +artifact_dir="$(cd "${2:?Updated flake artifact directory is required}" && pwd)" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ ! "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "A stable release tag is required" >&2 + exit 1 +fi +if [[ -n "$(git status --porcelain)" ]]; then + echo "Start with a clean working tree" >&2 + exit 1 +fi + +python3 - "$artifact_dir/release-candidate.json" "$tag" <<'PY' +import json, sys +from pathlib import Path +candidate = json.loads(Path(sys.argv[1]).read_text()) +if candidate['releaseTag'] != sys.argv[2] or candidate['version'] != sys.argv[2][1:]: + raise SystemExit('Flake artifact does not belong to the requested release') +PY +python3 "$script_dir/validate-release-version.py" --root "$artifact_dir" --version "${tag#v}" + +git fetch origin refs/heads/main:refs/remotes/origin/main +git fetch origin "refs/tags/$tag" +release_sha="$(git rev-parse 'FETCH_HEAD^{commit}')" +python3 - "$tag" <<'PY' +import re, subprocess, sys +cargo = subprocess.check_output(['git', 'show', 'origin/main:Cargo.toml'], text=True) +version = re.search(r'^version\s*=\s*"([0-9.]+)"', cargo, re.M)[1] +if tuple(map(int, sys.argv[1][1:].split('.'))) < tuple(map(int, version.split('.'))): + raise SystemExit('Refusing to prepare a release older than main') +PY + +branch="chore/promote-$tag" +git switch -c "$branch" origin/main +git merge --no-commit --no-ff "$release_sha" +cp "$artifact_dir/flake.nix" "$artifact_dir/flake.lock" . +python3 "$script_dir/validate-release-version.py" --version "${tag#v}" +# Check against Cargo too: overriding the expectation above is only for artifacts. +python3 "$script_dir/validate-release-version.py" +git add flake.nix flake.lock +if ! git diff --cached --quiet || [[ -f "$(git rev-parse --git-path MERGE_HEAD)" ]]; then + git commit -m "release: promote $tag sources and Nix packages together" +fi +echo "Prepared $branch. Review the candidate, push normally, and open one PR against main." diff --git a/scripts/publish-flake-test-branch.sh b/scripts/publish-flake-test-branch.sh new file mode 100755 index 0000000..d631315 --- /dev/null +++ b/scripts/publish-flake-test-branch.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Publish a tested snapshot flake beside its source branch; never open a main PR. +set -euo pipefail + +source_branch="${GITHUB_REF_NAME:?Run this workflow from the source branch}" +read -r branch expected version < <(python3 - <<'PYMETA' +import json, re +from pathlib import Path +metadata = json.loads(Path('release-candidate.json').read_text()) +if metadata['releaseTag'] != 'snapshot': + raise SystemExit('Only snapshot test branches can be published') +head = metadata['expectedTestHead'] +if head and not re.fullmatch('[0-9a-f]{40}', head): + raise SystemExit('Invalid test branch lease') +print(metadata['testBranch'], head or '-', metadata['version']) +PYMETA +) +if [[ "$expected" == - ]]; then + expected="" +fi +git check-ref-format "refs/heads/$branch" +python3 scripts/validate-release-version.py --version "$version" --release-tag snapshot +if git diff --quiet -- flake.nix flake.lock; then + echo "Snapshot flake files are already up to date" + exit 0 +fi + +# The lease was captured before candidate generation and Nix validation. +# Concurrent or delayed runs cannot replace a newer branch update. +git config user.name "github-actions[bot]" +git config user.email "41898282+github-actions[bot]@users.noreply.github.com" +git switch -c "$branch" +git add flake.nix flake.lock +git commit -m "test: prepare snapshot flake for $source_branch" +git push --force-with-lease="refs/heads/$branch:$expected" origin "HEAD:refs/heads/$branch" +echo "Published $branch from $source_branch for Nix testing" diff --git a/scripts/record-build-sources.py b/scripts/record-build-sources.py new file mode 100644 index 0000000..c45ecad --- /dev/null +++ b/scripts/record-build-sources.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Retain one source pair per build run so retries cannot move branch refs.""" + +import argparse +import json +import os +import re +import subprocess +from pathlib import Path + + +def record(path, gp, gpgui, output): + if path.exists(): + sources = json.loads(path.read_text()) + else: + sources = { + name: subprocess.check_output(["git", "-C", str(root), "rev-parse", "HEAD"], + text=True, stderr=subprocess.DEVNULL).strip() + for name, root in (("gp_sha", gp), ("gpgui_sha", gpgui)) + } + if set(sources) != {"gp_sha", "gpgui_sha"} or any( + not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{40}", value) + for value in sources.values() + ): + raise ValueError("invalid retained build source refs") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(sources, indent=2) + "\n") + with output.open("a") as stream: + for name, sha in sources.items(): + stream.write(f"{name}={sha}\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("path", type=Path) + parser.add_argument("--gp", type=Path, required=True) + parser.add_argument("--gpgui", type=Path, required=True) + args = parser.parse_args() + try: + record(args.path, args.gp, args.gpgui, Path(os.environ["GITHUB_OUTPUT"])) + except (OSError, ValueError, subprocess.CalledProcessError): + parser.exit(1, "Could not retain the selected build source refs.\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_release_flow.py b/scripts/tests/test_release_flow.py new file mode 100644 index 0000000..670bf2a --- /dev/null +++ b/scripts/tests/test_release_flow.py @@ -0,0 +1,218 @@ +import importlib.util +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +def load(name): + spec = importlib.util.spec_from_file_location(name, ROOT / 'scripts' / f'{name}.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +versions = load('validate-release-version') +sources = load('record-build-sources') + + +def git(root, *args): + return subprocess.check_output(['git', '-C', str(root), *args], text=True, + stderr=subprocess.DEVNULL).strip() + + +def initialize(root): + root.mkdir(parents=True, exist_ok=True) + git(root, 'init', '-b', 'main') + git(root, 'config', 'user.name', 'Release Test') + git(root, 'config', 'user.email', 'release@example.invalid') + + +def write_version(root, cargo='3.1.0', flake=None, tag=None): + flake = flake or cargo + (root / 'Cargo.toml').write_text(f'[workspace.package]\nversion = "{cargo}"\n') + (root / 'flake.nix').write_text(f'releaseVersion = "{flake}";\nreleaseTag = "{tag or "v" + flake}";\n') + (root / 'flake.lock').write_text('{}\n') + + +def commit(root, title): + git(root, 'add', '.') + git(root, 'commit', '-m', title) + return git(root, 'rev-parse', 'HEAD') + + +class ReleaseVersionTests(unittest.TestCase): + def test_main_rejects_old_new_and_snapshot_flakes(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + for selected, tag in [('3.1.0', 'v3.1.0'), ('3.3.0', 'v3.3.0'), ('3.2.0', 'snapshot')]: + write_version(root, '3.2.0', selected, tag) + with self.subTest(selected=selected, tag=tag), self.assertRaises(ValueError): + versions.validate(root) + write_version(root, '3.2.0') + self.assertEqual(versions.validate(root), '3.2.0') + + def test_snapshot_candidate_is_allowed_outside_main(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + write_version(root, '3.2.0', tag='snapshot') + self.assertEqual(versions.validate(root, release_tag='snapshot'), '3.2.0') + + def test_rollback_of_both_version_fields_is_rejected(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + initialize(root) + write_version(root, '3.3.0') + base = commit(root, 'new main') + write_version(root, '3.2.0') + with self.assertRaisesRegex(ValueError, 'older'): + versions.validate(root, base_ref=base) + + def test_actual_package_version_must_match(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + write_version(root, '3.2.0') + package = root / 'result' + (package / 'bin').mkdir(parents=True) + for name in ('gpclient', 'gpservice', 'gpauth', 'gpgui'): + tool = package / 'bin' / name + tool.write_text(f'#!/bin/sh\necho "{name} 3.2.0 (PRIVATE_IDENTITY)"\n') + tool.chmod(0o755) + versions.validate(root, package=package) + (package / 'bin/gpgui').write_text('#!/bin/sh\necho "gpgui 3.1.0 PRIVATE_DETAIL"\n') + with self.assertRaisesRegex(ValueError, 'gpgui'): + versions.validate(root, package=package) + + +class SourceSelectionTests(unittest.TestCase): + def test_retry_uses_original_sources_after_both_branches_advance(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + paths = [root / 'gp', root / 'gpgui'] + for repo in paths: + initialize(repo) + (repo / 'source').write_text('first') + commit(repo, 'first') + path, output = root / 'sources.json', root / 'output' + sources.record(path, *paths, output) + original = json.loads(path.read_text()) + for repo in paths: + (repo / 'source').write_text('second') + commit(repo, 'second') + sources.record(path, *paths, output) + self.assertEqual(json.loads(path.read_text()), original) + self.assertEqual(output.read_text().count(f"gp_sha={original['gp_sha']}"), 2) + + def test_corrupt_retained_sources_do_not_resolve_again(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + path = root / 'sources.json' + path.write_text('{"gp_sha":"main","gpgui_sha":"main"}') + with self.assertRaises(ValueError): + sources.record(path, root, root, root / 'output') + + +class PromotionTests(unittest.TestCase): + def test_sources_and_flake_are_committed_together_preserving_main_work(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + repo, origin, artifact = root / 'repo', root / 'origin.git', root / 'artifact' + initialize(repo) + write_version(repo) + old = commit(repo, '3.1') + git(repo, 'switch', '-c', 'release/next') + write_version(repo, '3.2.0', '3.1.0') + (repo / 'feature').write_text('released product feature') + release = commit(repo, '3.2 source') + git(repo, 'tag', 'v3.2.0') + git(repo, 'switch', 'main') + (repo / 'infrastructure').write_text('preserve newer main work') + main = commit(repo, 'main infrastructure') + git(root, 'init', '--bare', str(origin)) + git(repo, 'remote', 'add', 'origin', str(origin)) + git(repo, 'push', 'origin', 'main', 'v3.2.0') + artifact.mkdir() + write_version(artifact, '3.2.0') + (artifact / 'release-candidate.json').write_text(json.dumps({'version': '3.2.0', 'releaseTag': 'v3.2.0'})) + result = subprocess.run(['bash', str(ROOT / 'scripts/prepare-release-promotion.sh'), + 'v3.2.0', str(artifact)], cwd=repo, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(versions.validate(repo), '3.2.0') + self.assertEqual(git(repo, 'rev-parse', 'main'), main) + self.assertEqual(git(repo, 'rev-parse', 'v3.2.0'), release) + self.assertEqual(git(repo, 'show', f'{old}:Cargo.toml'), '[workspace.package]\nversion = "3.1.0"') + self.assertTrue((repo / 'infrastructure').exists()) + self.assertTrue((repo / 'feature').exists()) + self.assertEqual(git(repo, 'rev-list', '--parents', '-1', 'HEAD').split()[1:], [main, release]) + self.assertEqual(git(repo, 'status', '--porcelain'), '') + + +class SnapshotBranchTests(unittest.TestCase): + def run_publish(self, root, expected, version='3.2.0'): + write_version(root, version, tag='snapshot') + metadata = {'releaseTag': 'snapshot', 'version': version, + 'testBranch': 'chore/flake-feature-example', 'expectedTestHead': expected} + (root / 'release-candidate.json').write_text(json.dumps(metadata)) + env = dict(os.environ, GITHUB_REF_NAME='feature/example') + # CI checks out the same helper at its selected SHA. Use the real validator here. + (root / 'scripts').mkdir(exist_ok=True) + validator = root / 'scripts/validate-release-version.py' + validator.write_bytes((ROOT / 'scripts/validate-release-version.py').read_bytes()) + return subprocess.run(['bash', str(ROOT / 'scripts/publish-flake-test-branch.sh')], + cwd=root, env=env, capture_output=True, text=True) + + def test_branch_publication_and_stale_lease(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + repo, origin = root / 'repo', root / 'origin.git' + initialize(repo) + write_version(repo, '3.2.0') + base = commit(repo, 'source') + git(root, 'init', '--bare', str(origin)) + git(repo, 'remote', 'add', 'origin', str(origin)) + first = self.run_publish(repo, '') + self.assertEqual(first.returncode, 0, first.stdout + first.stderr) + head = git(repo, 'rev-parse', 'HEAD') + self.assertEqual(git(origin, 'rev-parse', 'refs/heads/chore/flake-feature-example'), head) + git(repo, 'switch', '--detach', base) + git(repo, 'branch', '-D', 'chore/flake-feature-example') + # Another run already created the branch after this run captured an absent head. + stale = self.run_publish(repo, '', '3.2.1') + self.assertNotEqual(stale.returncode, 0) + self.assertIn('stale info', stale.stderr) + self.assertEqual(git(origin, 'rev-parse', 'refs/heads/chore/flake-feature-example'), head) + git(repo, 'switch', '--detach', base) + git(repo, 'branch', '-D', 'chore/flake-feature-example') + fresh = self.run_publish(repo, head, '3.2.1') + self.assertEqual(fresh.returncode, 0, fresh.stdout + fresh.stderr) + self.assertNotEqual(git(origin, 'rev-parse', 'refs/heads/chore/flake-feature-example'), head) + self.assertEqual(git(origin, 'branch', '--list', 'main'), '') + + +class DockerMatrixTests(unittest.TestCase): + def test_workflow_matrix_follows_all_architecture_choices(self): + source = (ROOT / '.github/workflows/release.yaml').read_text() + block = source.split(' const inputs = ${{ toJson(inputs) }}\n', 1)[1].split('\n build:', 1)[0] + # Execute the actual github-script block, including both returned matrices. + lines = [] + for line in block.splitlines(): + if line.strip() and not line.startswith(' '): + break + lines.append(line[10:]) + js = '\n'.join(lines) + for arch, expected in [('all', ['linux/amd64', 'linux/arm64']), + ('x86_64', ['linux/amd64']), ('arm64', ['linux/arm64'])]: + harness = 'const core={setOutput:(k,v)=>console.log(v)}; const inputs=' + json.dumps({'arch': arch}) + ';\n' + result = subprocess.check_output(['node', '-e', harness + '(function(){\n' + js + '\n})();'], text=True) + matrix = json.loads(result.splitlines()[0]) + self.assertEqual([row['platform'] for row in matrix['include']], expected) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/validate-release-version.py b/scripts/validate-release-version.py new file mode 100644 index 0000000..de261c0 --- /dev/null +++ b/scripts/validate-release-version.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Check the release selected by a flake and, optionally, its installed tools.""" + +import argparse +import re +import subprocess +from pathlib import Path + + +def cargo_version(root, cargo=None): + if cargo is None: + cargo = (root / "Cargo.toml").read_text() + workspace = re.search(r"(?ms)^\[workspace\.package\]\s*\n(.*?)(?=^\[|\Z)", cargo) + if workspace is None: + raise ValueError("Cargo.toml has no workspace.package section") + return field(workspace[1], "version") + + +def field(text, name): + matches = re.findall(rf'^\s*{name}\s*=\s*"([^"]+)"\s*;?\s*$', text, re.M) + if len(matches) != 1: + raise ValueError(f"expected one {name} declaration") + return matches[0] + + +def validate(root, version=None, release_tag=None, package=None, base_ref=None): + expected = version or cargo_version(root) + if not re.fullmatch(r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)", expected): + raise ValueError("expected a stable X.Y.Z version") + tag = release_tag or f"v{expected}" + if tag not in (f"v{expected}", "snapshot"): + raise ValueError("release tag does not match the expected version") + flake = (root / "flake.nix").read_text() + if field(flake, "releaseVersion") != expected or field(flake, "releaseTag") != tag: + raise ValueError(f"flake must select {tag} at version {expected}") + if base_ref: + cargo = subprocess.check_output(["git", "-C", str(root), "show", f"{base_ref}:Cargo.toml"], + text=True, stderr=subprocess.DEVNULL) + base = cargo_version(root, cargo) + if tuple(map(int, expected.split("."))) < tuple(map(int, base.split("."))): + raise ValueError("candidate release is older than the main base") + if package: + for name in ("gpclient", "gpservice", "gpauth", "gpgui"): + result = subprocess.run([str(package / "bin" / name), "--version"], + capture_output=True, text=True, check=True) + # Keep private source identity and other command output out of logs. + match = re.match(rf"{name}\s+(\d+\.\d+\.\d+)(?:\s|$)", result.stdout) + if match is None or match[1] != expected: + raise ValueError(f"installed {name} does not report version {expected}") + return expected + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(".")) + parser.add_argument("--version", help="Explicit candidate version; defaults to Cargo.toml") + parser.add_argument("--release-tag", help="Explicit candidate tag; defaults to v") + parser.add_argument("--package", type=Path) + parser.add_argument("--base-ref", help="Reject a release older than this main commit") + args = parser.parse_args() + try: + version = validate(args.root, args.version, args.release_tag, args.package, args.base_ref) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + # CalledProcessError does not include captured stdout/stderr in its string. + parser.exit(1, f"Release version check failed: {error}\n") + print(f"Verified release version {version}") + + +if __name__ == "__main__": + main()