Add upload_to_gist and artifact_from_directory (#8)

Co-authored-by: Mosè Giordano <[email protected]>
JuliaPackaging · Dec 13, 2021 · 0dc40ee · 0dc40ee
1 parent f22ac24
commit 0dc40ee
Show file tree

Hide file tree

Showing 7 changed files with 433 additions and 5 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         version:
-          - '1.3'
+          - '1.6'
           - '1'
           - 'nightly'
         os:

diff --git a/Project.toml b/Project.toml
@@ -5,11 +5,18 @@ version = "0.1.1"
 
 [deps]
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
+HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+gh_cli_jll = "5d31d589-30fb-542f-b82d-10325e863e38"
 
 [compat]
-julia = "1.3"
+Git = "1"
+HTTP = "0.9"
+gh_cli_jll = "2"
+julia = "1.6"
 
 [extras]
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

diff --git a/README.md b/README.md
@@ -39,3 +39,60 @@ julia> run(`ls $ans`);
 JuliaMono-Black.ttf	JuliaMono-Bold.ttf	 JuliaMono-Light.ttf	JuliaMono-RegularLatin.ttf  LICENSE
 JuliaMono-BoldLatin.ttf  JuliaMono-ExtraBold.ttf  JuliaMono-Medium.ttf	JuliaMono-Regular.ttf
 ```
+
+### Archive a directory and upload it to gist
+
+You can create an artifact from a directory using `artifact_from_directory` and
+then upload it to gist with `upload_to_gist`.  Note that `upload_to_gist`
+requires login with the [GitHub CLI `gh`](https://github.com/cli/cli).
+
+```julia
+julia> using ArtifactUtils
+
+julia> tempdir = mktempdir();
+
+julia> write(joinpath(tempdir, "file"), "hello");
+
+julia> artifact_id = artifact_from_directory(tempdir)
+SHA1("538e83d637ab07ada6d841aa2454e0d5af4e52b3")
+
+julia> gist = upload_to_gist(artifact_id)
+- Creating gist...
+✓ Created gist
+Cloning into '.'...
+remote: Enumerating objects: 3, done.
+remote: Counting objects: 100% (3/3), done.
+remote: Total 3 (delta 0), reused 0 (delta 0), pack-reused 0
+Receiving objects: 100% (3/3), done.
+Switched to a new branch '__tmp__'
+[__tmp__ (root-commit) f5a58a9] Initial commit
+Switched to branch 'master'
+Your branch is up to date with 'origin/master'.
+HEAD is now at f5a58a9 Initial commit
+[master 44dfe9a] Add files
+ 1 file changed, 0 insertions(+), 0 deletions(-)
+ create mode 100644 538e83d637ab07ada6d841aa2454e0d5af4e52b3.tar.gz
+Counting objects: 5, done.
+Delta compression using up to 128 threads.
+Compressing objects: 100% (4/4), done.
+Writing objects: 100% (5/5), 455 bytes | 455.00 KiB/s, done.
+Total 5 (delta 1), reused 0 (delta 0)
+remote: Resolving deltas: 100% (1/1), done.
+To gist.github.com:a9ceed430ff970412fc6606ef1b84b6a
+ + 2668e40...44dfe9a master -> master (forced update)
+upload_to_gist(SHA1("538e83d637ab07ada6d841aa2454e0d5af4e52b3")) →
+
+[538e83d637ab07ada6d841aa2454e0d5af4e52b3]
+git-tree-sha1 = "538e83d637ab07ada6d841aa2454e0d5af4e52b3"
+
+    [[538e83d637ab07ada6d841aa2454e0d5af4e52b3.download]]
+    sha256 = "a530e9f7e371eeea4aa4fbce83a00ed32233b7766314670b1c0779eb46a7b68d"
+    url = "https://gist.github.com/tkf/a9ceed430ff970412fc6606ef1b84b6a/raw/538e83d637ab07ada6d841aa2454e0d5af4e52b3.tar.gz"
+```
+
+You can copy-and-paste the printed artifact fragment into your `Artifacts.toml`
+file.  You can also call `add_artifact!` with the `gist` result object.
+
+```julia
+julia> add_artifact!("Artifacts.toml", "hello_world", gist)
+```
diff --git a/docs/src/index.md b/docs/src/index.md
@@ -6,6 +6,8 @@ CurrentModule = ArtifactUtils
 
 ```@index
 add_artifact!
+artifact_from_directory
+upload_to_gist
 ```
 
 ```@autodocs

diff --git a/src/ArtifactUtils.jl b/src/ArtifactUtils.jl
@@ -1,12 +1,19 @@
 module ArtifactUtils
 
+import Git
+import HTTP
+import TOML
+import gh_cli_jll
 using Pkg.Artifacts
 using Pkg.PlatformEngines
 using Pkg.GitTools
+using Base: SHA1
 using SHA
 using Downloads: download
 
-export add_artifact!
+export add_artifact!, artifact_from_directory, upload_to_gist
+
+include("gistutils.jl")
 
 function sha256sum(tarball_path)
     return open(tarball_path, "r") do io
@@ -44,10 +51,11 @@ function add_artifact!(
     artifacts_toml::String,
     name::String,
     tarball_url::String;
-    clear=true,
+    clear = true,
     options...,
 )
-    @static isdefined(PlatformEngines, :probe_platform_engines!) && probe_platform_engines!()
+    @static isdefined(PlatformEngines, :probe_platform_engines!) &&
+            probe_platform_engines!()
 
     tarball_path = download(tarball_url)
     sha256 = sha256sum(tarball_path)
@@ -70,4 +78,173 @@ function add_artifact!(
     return git_tree_sha1
 end
 
+"""
+    artifact_from_directory(source; follow_symlinks = false) -> artifact_id::SHA1
+
+Create an artifact from the `source` directory and return the `artifact_id`.
+"""
+artifact_from_directory(source; follow_symlinks::Bool = false) =
+    create_artifact() do artifact_dir
+        cp(source, artifact_dir; force = true, follow_symlinks = follow_symlinks)
+        # Manually copy the executable bit in Windows:
+        # https://github.com/simeonschaub/ArtifactUtils.jl/pull/8#discussion_r767210865
+        Sys.iswindows() && copy_mode(source, artifact_dir)
+    end
+
+"""
+    copy_mode(src, dst)
+
+Copy mode of `src` to `dst` recursively in a way compatible to how `Pkg` (Git)
+compute the tree hash.
+
+See `Pkg.GitTools.gitmode` comment for how the file mode is handled specially in Windows:
+https://github.com/JuliaLang/Pkg.jl/blob/247a4062bfde19d93bdcbaccc9737df496fd0c2b/src/GitTools.jl#L189-L192
+
+`copy_mode` is based on:
+https://github.com/JuliaIO/Tar.jl/blob/6a946029685639b69ce5a7cc4c4a6c0e6c6b2697/src/extract.jl#L145-L153
+"""
+function copy_mode(src, dst)
+    if !isdir(dst)
+        chmod(dst, UInt(GitTools.gitmode(string(src))))
+        return
+    end
+    for name in readdir(dst)
+        sub_src = joinpath(src, name)
+        sub_dst = joinpath(dst, name)
+        copy_mode(sub_src, sub_dst)
+    end
+end
+
+struct GistUploadResult
+    artifact_id::SHA1
+    filename::String
+    localpath::Union{String,Nothing}
+    url::String
+    sha256::String
+    private::Bool
+end
+
+"""
+    upload_to_gist(
+        artifact_id::SHA1,
+        [tarball];
+        private::Bool = true,
+        honor_overrides = false,
+        # Following options are aviailable only when `tarball` is not specified:
+        name::AbstractString = "\$artifact_id.tar.gz",
+        extension::AbstractString = ".tar.gz",
+    ) -> gist
+
+Create an artifact archive at path `tarball` (or in a temporary location) and upload it to
+gist. The returned value `gist` can be passed to `add_artifact!`.
+
+# Extended help
+
+## Examples
+```julia
+using ArtifactUtils
+add_artifact!("Artifact.toml", "name", upload_to_gist(artifact_from_directory("source")))
+```
+
+creates an artifact from files in the `"source"` directory, upload it to gist, and then
+add it to `"Artifact.toml"` with the name `"name"`.
+
+## Keyword Arguments
+- `private`: if `true`, upload the archive to a private gist
+- `name`: name of the archive file, including file extension
+- `extension`: file extension of the tarball. It can be used for specifying the compression
+  method.
+- `honor_overrides`: see `Pkg.Artifacts.archive_artifact`
+"""
+function upload_to_gist end
+
+function upload_to_gist(
+    artifact_id::SHA1,
+    tarball::AbstractString;
+    private::Bool = true,
+    archive_options...,
+)
+    mkpath(dirname(tarball))
+    archive_artifact(artifact_id, tarball; archive_options...)
+    sha256 = sha256sum(tarball)
+    url = gist_from_file(tarball; private = private)
+    return GistUploadResult(
+        artifact_id,
+        basename(tarball),
+        abspath(tarball),
+        url,
+        sha256,
+        private,
+    )
+end
+
+function upload_to_gist(
+    artifact_id::SHA1;
+    name::Union{AbstractString,Nothing} = nothing,
+    extension::Union{AbstractString,Nothing} = nothing,
+    options...,
+)
+    if name !== nothing && extension !== nothing
+        error(
+            "Options `name` and `extension` are mutually exclusive. Got: name = ",
+            name,
+            " extension = ",
+            extension,
+        )
+    end
+
+    tarball = if name === nothing
+        string(artifact_id, something(extension, ".tar.gz"))
+    else
+        name
+    end
+
+    return mktempdir() do dir
+        upload_to_gist(artifact_id, joinpath(dir, tarball); options...)
+    end
+end
+
+function add_artifact!(
+    artifacts_toml::String,
+    name::String,
+    gist::GistUploadResult;
+    options...,
+)
+    bind_artifact!(
+        artifacts_toml,
+        name,
+        gist.artifact_id;
+        download_info = [(gist.url, gist.sha256)],
+        options...,
+    )
+end
+
+print_artifact_entry(gist::GistUploadResult; options...) =
+    print_artifact_entry(stdout, gist; options...)
+function print_artifact_entry(
+    io::IO,
+    gist::GistUploadResult;
+    name::AbstractString = replace(gist.filename, r"\.tar.[^\.]*$" => ""),
+)
+    dict = Dict(
+        name => Dict(
+            "git-tree-sha1" => string(gist.artifact_id),
+            "download" => [Dict("url" => gist.url, "sha256" => gist.sha256)],
+        ),
+    )
+    TOML.print(io, dict)
+end
+
+function Base.show(io::IO, ::MIME"text/plain", gist::GistUploadResult)
+    print(io, upload_to_gist, "(")
+    show(io, gist.artifact_id)
+    if !gist.private
+        print(io, "; private = false")
+    end
+    println(io, ") →")
+    println(io)
+
+    print(io, strip(sprint(print_artifact_entry, gist)))
+end
+
 end