Initial PR-based artifact update workflow

.github/workflows/CI.yml

@@ -4,15 +4,47 @@ on:
   push:
     branches:
       - main
+    paths:
+      - "!Artifacts.toml"  # Impending package release. Will trigger via `tags`
     tags: ["*"]
 concurrency:
   # Skip intermediate builds: on all builds except on the "main" branch.
   # Cancel intermediate builds: only if it is a pull request build.
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.run_number || 0 }}
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 jobs:
+  unpublished:
+    name: Artifact Check
+    runs-on: ubuntu-latest
+    outputs:
+      key: ${{ steps.key.outputs.key }}
+      tarball_filename: ${{ steps.key.outputs.tarball_filename }}
+      content_hash: ${{ steps.key.outputs.content_hash }}
+    steps:
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            modified_artifacts_toml:
+              - Artifacts.toml
+      - uses: actions/checkout@v4
+        if: ${{ steps.filter.outputs.modified_artifacts_toml == 'true' }}
+      - name: Check for unpublished artifact
+        if: ${{ steps.filter.outputs.modified_artifacts_toml == 'true' }}
+        id: key
+        shell: julia --color=yes {0}
+        run: |
+          include(joinpath(pwd(), "gen", "artifacts.jl"))
+          (; key, tarball_filename, content_hash) = gh_artifact()
+          open(ENV["GITHUB_OUTPUT"], "a") do io
+              println(io, "key=$key")
+              println(io, "tarball_filename=$tarball_filename")
+              println(io, "content_hash=$content_hash")
+          end
+
   test:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    needs: unpublished
     runs-on: ${{ matrix.os }}
     continue-on-error: ${{ matrix.version == 'nightly' }}
     strategy:
@@ -28,6 +60,51 @@ jobs:
           - x64
     steps:
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # Allows checkout of Artifacts.toml from base ref
+      - uses: dawidd6/action-download-artifact@v3
+        if: ${{ needs.unpublished.outputs.key }}
+        id: action-artifact
+        with:
+          workflow: Update.yaml
+          name: ${{ needs.unpublished.outputs.key }}
+      - name: Clear cached Overrides.toml
+        if: ${{ !needs.unpublished.outputs.key }}
+        run: rm -f ~/.julia/artifacts/Overrides.toml
+      - name: Create artifacts Overrides.toml
+        if: ${{ needs.unpublished.outputs.key }}
+        run: |
+          set -ex
+          tarball_path="$PWD/${{ needs.unpublished.outputs.tarball_filename }}"
+          [ -f "$tarball_path" ] || exit 1
+
+          # A valid Artifacts.toml file is required: Pkg.jl issue #2662
+          mv Artifacts.toml NewArtifacts.toml
+          git checkout origin/${{ github.base_ref }} -- Artifacts.toml
+
+          artifact_dir="$HOME/.julia/artifacts/${{ needs.unpublished.outputs.content_hash }}"
+          mkdir -p "$artifact_dir"
+          tar xvf "$tarball_path" -C "$artifact_dir"
+
+          # Why doesn't this work?
+          # cat > ~/.julia/artifacts/Overrides.toml <<EOF
+          # [dc5dba14-91b3-4cab-a142-028a31da12f7]
+          # tzjdata = "${{ needs.unpublished.outputs.content_hash }}"
+          # EOF
+
+          cat > ~/.julia/artifacts/Overrides.toml <<EOF
+          [dc5dba14-91b3-4cab-a142-028a31da12f7]
+          tzjdata = "$artifact_dir"
+          EOF
+
+          echo "TZJDATA_ARTIFACT_TOML=$PWD/NewArtifacts.toml" | tee -a "$GITHUB_ENV"
+          echo "TZJDATA_TARBALL_PATH=$tarball_path" | tee -a "$GITHUB_ENV"
+      - name: Debug
+        if: ${{ needs.unpublished.outputs.key }}
+        run: |
+          set -x
+          cat ~/.julia/artifacts/Overrides.toml
+          cat Artifacts.toml
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}

.github/workflows/Release.yaml

@@ -0,0 +1,50 @@
+name: Release
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "Artifacts.toml"
+jobs:
+  publish:
+    name: Publish
+    runs-on: ubuntu-latest
+    # These permissions are needed to:
+    # - Create a GitHub release: https://github.com/ncipollo/release-action#notes
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Determine artifact details
+        id: details
+        shell: julia --color=yes {0}
+        run: |
+          using Pkg.Types: read_project
+          include(joinpath(pwd(), "gen", "artifacts.jl"))
+          (; key, tarball_filename) = gh_artifact()
+          project = read_project("Project.toml")
+          open(ENV["GITHUB_OUTPUT"], "a") do io
+              println(io, "key=$key")
+              println(io, "tarball_filename=$tarball_filename")
+              println(io, "tag=v$(project.version)")
+          end
+      - uses: dawidd6/action-download-artifact@v3
+        id: action-artifact
+        with:
+          workflow: Update.yaml
+          name: ${{ steps.details.outputs.key }}
+      # As `ncipollo/release-action`'s `artifactErrorsFailBuild` input will still cause a release
+      # to be created we'll perform this check to fail earlier.
+      - name: Validate artifact retrieved
+        run: |
+          [ -f "${{ steps.details.outputs.tarball_filename }}" ] || exit 1
+      - name: Build Changelog
+        id: build_changelog
+        uses: mikepenz/release-changelog-builder-action@v4
+      - name: Publish Release
+        uses: ncipollo/release-action@v1
+        with:
+          tag: ${{ steps.details.outputs.tag }}
+          body: ${{ steps.build_changelog.outputs.changelog }}
+          artifacts: ${{ steps.details.outputs.tarball_filename }}
+          artifactErrorsFailBuild: true

.github/workflows/Update.yaml

@@ -0,0 +1,65 @@
+---
+name: Update
+on:
+  schedule:
+    - cron: "*/5 * * * *"  # Every 5 minutes (for now)
+jobs:
+  artifacts:
+    name: Artifacts
+    runs-on: ubuntu-latest
+    # These permissions are needed to:
+    # - Create PRs: https://github.com/marketplace/actions/create-pull-request#workflow-permissions
+    permissions:
+      contents: write
+      pull-requests: write
+    env:
+      JULIA_PROJECT: gen
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || 'main' }}
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: "1"
+      - uses: julia-actions/cache@v1
+      - name: Instantiate
+        shell: julia --color=yes {0}
+        run: |
+          using Pkg
+          Pkg.instantiate()
+      - name: Build tzdata
+        id: build
+        shell: julia --color=yes {0}
+        run: |
+          include(joinpath(pwd(), "gen", "make.jl"))
+          (; tarball_path, tarball_sha256, new_version, commit_message) = update_tzdata()
+          key = basename(tarball_path) * "-" * tarball_sha256
+          @show key tarball_path commit_message
+          open(ENV["GITHUB_OUTPUT"], "a") do io
+              println(io, "key=$key")
+              println(io, "tarball_path=$tarball_path")
+              println(io, "commit_message=$commit_message")
+          end
+          println("workflow_ref=${{ github.workflow_ref }}")
+      - run: git diff
+      # Store the Julia artifact tarball as a GitHub actions artifact. This will allow us to retrieve this
+      # tarball from other workflows.
+      - uses: actions/upload-artifact@v4
+        id: action-artifact
+        with:
+          name: ${{ steps.build.outputs.key }}
+          path: ${{ steps.build.outputs.tarball_path }}
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          base: main  # Shouldn't be required for `workflow_dispatch`
+          title: ${{ steps.build.outputs.commit_message }}
+          body: >-
+            An automated PR generated by the ${{ github.workflow }} workflow.
+          add-paths: |
+            Project.toml
+            Artifacts.toml
+          commit-message: ${{ steps.build.outputs.commit_message }}
+          branch: gh/update-tzdata
+          delete-branch: ${{ github.event_name == 'pull_request' }}
+          token: ${{ secrets.TZJDATA_UPDATE_TOKEN }}  # TODO: Fine-grained token expires

Project.toml

@@ -7,12 +7,20 @@ version = "1.0.0+2023c"
 Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 
 [compat]
+CodecZlib = "0.7"
+SHA = "1.6"
+Tar = "1"
 TimeZones = "1"
 julia = "1.6"
 
 [extras]
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 
 [targets]
-test = ["TimeZones", "Test"]
+test = ["CodecZlib", "Pkg", "SHA", "TOML", "Tar", "Test", "TimeZones"]

gen/artifacts.jl

@@ -0,0 +1,19 @@
+using TOML: TOML
+
+const PKG_ROOT = joinpath(@__DIR__(), "..")
+
+function gh_artifact(artifact_toml=joinpath(PKG_ROOT, "Artifacts.toml"))
+    toml = TOML.parsefile(artifact_toml)
+    tarball_artifact = only(toml["tzjdata"]["download"])
+    tarball_filename = basename(tarball_artifact["url"])
+    tarball_sha256 = tarball_artifact["sha256"]
+    content_hash = toml["tzjdata"]["git-tree-sha1"]
+
+    key = "$(tarball_filename)-$(tarball_sha256)"
+    return (;
+        tarball_filename,
+        tarball_sha256,
+        content_hash,
+        key,
+    )
+end

gen/make.jl

@@ -109,35 +109,44 @@ function upload_to_github_release(owner, repo_name, commit, tag, path; token=ENV
     run(cmd)
 end
 
-# TODO: Re-running always bumps version
-if abspath(PROGRAM_FILE) == @__FILE__
-    tzdata_version = tzdata_latest_version()
-    tarball_name = "tzjfile-v1-tzdata$(tzdata_version).tar.gz"
-
-    # Build tzjfile artifact content
-    # TZData.cleanup(tzdata_version, _scratch_dir())
-    compiled_dir = TZData.build(tzdata_version, _scratch_dir())
-
-    @info "Creating tarball $tarball_name"
-    tarball_path = joinpath(tempdir(), tarball_name)
-    create_tarball(compiled_dir, tarball_path)
-
+function update_tzdata()
     repo_path = joinpath(@__DIR__, "..")
     pkg_url = remote_url(repo_path)
 
     # Read Project.toml
     project_toml = joinpath(repo_path, "Project.toml")
     project = read_project(project_toml)
     old_version = project.version
-    new_version = Base.nextpatch(project.version)
+    old_tzdata_version = only(old_version.build)
+
+    # Always fetch the current list of tzdata versions (ignoring any caching).
+    tzdata_versions = TZData.tzdata_versions()
+    i = findfirst(==(old_tzdata_version), tzdata_versions)
+    if i == length(tzdata_versions)
+        new_tzdata_version = tzdata_versions[i]
+        new_version = Base.nextpatch(old_version)
+    else
+        new_tzdata_version = tzdata_versions[i + 1]
+        new_version = Base.nextminor(old_version)
+    end
+
+    tarball_name = "tzjfile-v1-tzdata$(new_tzdata_version).tar.gz"
+
+    # Build tzjfile artifact content
+    # TZData.cleanup(new_tzdata_version, _scratch_dir())
+    compiled_dir = TZData.build(new_tzdata_version, _scratch_dir())
+
+    @info "Creating tarball $tarball_name"
+    tarball_path = joinpath(tempdir(), tarball_name)
+    create_tarball(compiled_dir, tarball_path)
 
     # Include tzdata version in build number
     new_version = VersionNumber(
         new_version.major,
         new_version.minor,
         new_version.patch,
         (),
-        (tzdata_version,),
+        (new_tzdata_version,),
     )
 
     @info "Bumping package $(project.name) from $old_version -> $new_version"
@@ -148,18 +157,49 @@ if abspath(PROGRAM_FILE) == @__FILE__
     artifact_url = "$(pkg_url)/releases/download/$(tag)/$(basename(tarball_path))"
 
     artifacts_toml = joinpath(repo_path, "Artifacts.toml")
+    content_hash = tree_hash_sha1(tarball_path)
+    tarball_sha256 = sha256sum(tarball_path)
     bind_artifact!(
         artifacts_toml,
         "tzjdata",
-        tree_hash_sha1(tarball_path);
-        download_info=[(artifact_url, sha256sum(tarball_path))],
+        content_hash;
+        download_info=[(artifact_url, tarball_sha256)],
         force=true,
     )
 
+    commit_message = "Set artifact to tzdata$(new_tzdata_version) and project to $(new_version)"
+
+    return (;
+        repo_path,
+        project_toml,
+        artifacts_toml,
+        artifact_url,
+        tarball_path,
+        tarball_sha256,
+        old_tzdata_version,
+        new_tzdata_version,
+        old_version,
+        new_version,
+        commit_message,
+    )
+end
+
+# TODO: Re-running always bumps version
+if abspath(PROGRAM_FILE) == @__FILE__
+    (;
+        repo_path,
+        new_tzdata_version,
+        new_version,
+        project_toml,
+        artifacts_toml,
+        artifact_url,
+        tarball_path,
+        commit_message,
+    ) = update_tzdata()
+
     # TODO: Ensure no other files are staged before committing
     @info "Committing and pushing Project.toml and Artifacts.toml"
     branch = "main"
-    message = "Set artifact to tzdata$(tzdata_version) and project to $(new_version)"
 
     # TODO: ghr and LibGit2 use different credential setups. Double check
     # what BB does here.
@@ -168,7 +208,7 @@ if abspath(PROGRAM_FILE) == @__FILE__
             # TODO: This allows empty commits
             LibGit2.add!(repo, basename(artifacts_toml))
             LibGit2.add!(repo, basename(project_toml))
-            LibGit2.commit(repo, message)
+            LibGit2.commit(repo, commit_message)
 
             # Same as "refs/heads/$branch" but fails if branch doesn't exist locally
             branch_ref = LibGit2.lookup_branch(repo, branch)