diff --git a/.github/workflows/check_changelog.yml b/.github/workflows/check_changelog.yml new file mode 100644 index 0000000..5b756c9 --- /dev/null +++ b/.github/workflows/check_changelog.yml @@ -0,0 +1,13 @@ +name: Check Changelog + +on: + pull_request: + types: [opened, reopened, edited, synchronize] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Check that CHANGELOG is touched + run: | + cat $GITHUB_EVENT_PATH | jq .pull_request.title | grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' || git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..8857f67 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,45 @@ +name: Release +on: + release: + types: + - published +jobs: + register: + name: Package, Publish, and Register + runs-on: + - ubuntu-latest + steps: + - id: checkout + name: Checkout code + uses: actions/checkout@v2 + - if: ${{ github.event_name != 'pull_request' || ! github.event.pull_request.head.repo.fork }} + name: Login to GitHub Package Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GHCR_TOKEN }} + - id: setup-pack + uses: buildpacks/github-actions/setup-pack@v4.1.0 + - id: package + run: | + #!/usr/bin/env bash + set -euo pipefail + BP_ID="$(cat buildpack.toml | yj -t | jq -r .buildpack.id)" + VERSION="$(cat buildpack.toml | yj -t | jq -r .buildpack.version)" + PACKAGE="${REPO}/$(echo "$BP_ID" | sed 's/\//_/g')" + pack buildpack package --publish ${PACKAGE}:${VERSION} + DIGEST="$(crane digest ${PACKAGE}:${VERSION})" + echo "::set-output name=bp_id::$BP_ID" + echo "::set-output name=version::$VERSION" + echo "::set-output name=address::${PACKAGE}@${DIGEST}" + shell: bash + env: + REPO: ghcr.io/${{ github.repository_owner }}/buildpacks + - id: register + uses: docker://ghcr.io/buildpacks/actions/registry/request-add-entry:4.1.0 + with: + token: ${{ secrets.PUBLIC_REPO_TOKEN }} + id: ${{ steps.package.outputs.bp_id }} + version: ${{ steps.package.outputs.version }} + address: ${{ steps.package.outputs.address }} \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt index beb6d32..d0d2fe6 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2014 Paul Hammond +Copyright (c) 2021 Paulo Fagiani Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index dcb7603..772f352 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,51 @@ -# S3 Tarball Buildpack +# S3 Tarball Cloud Native Buildpack -This is a [Heroku Buildpack](https://devcenter.heroku.com/articles/buildpacks) +This is a [Cloud Native Buildpack](https://buildpacks.io/docs/concepts/components/buildpack/) that can download tarballs from private [Amazon S3](http://aws.amazon.com/s3/) -buckets. It gives you a way of deploying pre-built code to -[Heroku](http://www.heroku.com/) without making it publicly accessible. +buckets. It gives you a way of adding private files outside the main git repository such +as certificates, and more complex attributes that won't fit on environment variables to +the container at build time without making it publicly accessible. -## Usage - - $ heroku config:add BUILDPACK_URL=https://github.com/paulhammond/s3-tarball-buildpack.git - - $ cat .buildpack-s3-tarballs - AWS_ACCESS_KEY_ID=AKIA0000000000000000 - AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - s3://bucket/path/to/tarball.tgz - s3://bucket/path/to/somethingelse.tgz +## How it works -You probably want to use an [IAM key](http://aws.amazon.com/iam/) with limited -access. This code only requires `s3:GetObject` access to files. +This buildpack aims to allow you to write files in build time in any path within the root application +directory (`/app` or `/workspace` which is an alias). Therefore, with a tar archive you can achieve that +by defining the paths desired that will be expanded when downloaded. A second optional benefit is compression +when your archives have a significant size they can benefit of a faster download. -If you don't want to check your IAM keys into revision control, you can store -them in Heroku's config system. Keys specified in the .buildpack-s3-tarballs -file have precedence over keys in the config system. +## Usage - $ heroku config:add BUILDPACK_URL=https://github.com/paulhammond/s3-tarball-buildpack.git - $ heroku config:add AWS_ACCESS_KEY_ID=AKIA0000000000000000 - $ heroku config:add AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + $ cat < S3file + s3://my-private-bucket/path/to/tarball.tgz + s3://my-other-bucket/path/to/somethingelse.tgz + http://my-public-domain.com/tarball.tgz + https://my-other-public-domain.com/path/theother.tgz + EOF - $ cat .buildpack-s3-tarballs - s3://bucket/path/to/tarball.tgz + $ pack build my-app --builder heroku/buildpacks:20 --buildpack fagiani/s3-tarball-buildpack \ + --env AWS_ACCESS_KEY_ID=AKIA000000000000000 --env AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxx ... -In most cases you'll use this buildpack in conjunction with other buildpacks -using [heroku-buildpack-multi](https://github.com/ddollar/heroku-buildpack-multi): +> Alternatively you can use `S3_AWS_ACCESS_KEY_ID` and `S3_AWS_SECRET_ACCESS_KEY` to avoid IAM +> conflicts when using AWS containers to run `pack build` - $ heroku config:add BUILDPACK_URL=https://github.com/ddollar/heroku-buildpack-multi.git +You probably want to use an [IAM key](http://aws.amazon.com/iam/) with limited +access. This code only requires `s3:GetObject` access to files. - $ cat .buildpack-s3-tarballs - AWS_ACCESS_KEY_ID=AKIA0000000000000000 - AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - s3://bucket/path/to/tarball.tgz +In most cases you'll use this buildpack in conjunction with other buildpacks. - $ cat .buildpacks - https://github.com/paulhammond/s3-tarball-buildpack.git - https://github.com/ryandotsmith/null-buildpack.git +> Please notice that public tarball URLs are also accepted and for that no credentials are required. ## See also + * [s3-tarball-buildpack](https://github.com/paulhammond/s3-tarball-buildpack) * [heroku-buildpack-vendorbinaries](https://github.com/peterkeen/heroku-buildpack-vendorbinaries) * [s3simple](https://github.com/paulhammond/s3simple) * [Heroku Slug API](https://blog.heroku.com/archives/2013/12/20/programmatically_release_code_to_heroku_using_the_platform_api) +## Contributing + +Feel free to contribute by opening a issue or sending a PR. + ## Licence MIT license, see LICENSE.txt for details. diff --git a/bin/build b/bin/build new file mode 100755 index 0000000..d86aa3d --- /dev/null +++ b/bin/build @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -euo pipefail + +env_dir="$2/env" + +function error() { + echo " ! $*" >&2 + exit 1 +} + +function topic() { + echo "-----> $*" +} + +# s3simple is a small, simple bash s3 client with minimal dependencies. +# See http://github.com/paulhammond/s3simple for documentation and licence. +s3simple() { + local url="$1" + local file="${2:--}" + + if [ "${url:0:5}" != "s3://" ]; then + error "Downloadable tarballs must be a s3:// compatible URL" + fi + local path="${url:4}" + + if [ -z "${AWS_ACCESS_KEY_ID-}" ] || [ -z "${S3_AWS_ACCESS_KEY_ID-}" ]; then + error "AWS_ACCESS_KEY_ID or S3_AWS_ACCESS_KEY_ID must be set in order to download a S3 tarball" + fi + + if [ -z "${AWS_SECRET_ACCESS_KEY-}" ] || [ -z "${S3_AWS_SECRET_ACCESS_KEY-}"]; then + error "AWS_SECRET_ACCESS_KEY or S3_AWS_SECRET_ACCESS_KEY must be set in order to download a S3 tarball" + fi + + local method md5 args + method="GET" + md5="" + args="-o $file" + + local date="$(date -u '+%a, %e %b %Y %H:%M:%S +0000')" + local string_to_sign + printf -v string_to_sign "%s\n%s\n\n%s\n%s" "$method" "$md5" "$date" "$path" + local signature=$(echo -n "$string_to_sign" | openssl sha1 -binary -hmac "${AWS_SECRET_ACCESS_KEY}" | openssl base64) + local authorization="AWS ${AWS_ACCESS_KEY_ID}:${signature}" + + curl $args -s -f -H Date:"${date}" -H Authorization:"${authorization}" https://s3.amazonaws.com"${path}" +} + +httpsimple() { + local url="$1" + local file="${2:--}" + + if [ "${url:0:7}" != "http://" ] || [ "${url:0:8}" != "https://"]; then + error "Downloadable tarballs need to be a regular URL" + fi + + local args="-o $file" + + curl $args -s -f "${url}" +} + +for e in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY S3_AWS_ACCESS_KEY_ID S3_AWS_SECRET_ACCESS_KEY; do + if [ -f $env_dir/$e ]; then + export "$e=$(cat $env_dir/$e)" + fi +done + +while read line; do + case "$line" in + s3://*) + topic "Downloading and extracting $line from S3 bucket" + s3simple "$line" | tar -xv + ;; + http*://*) + topic "Downloading and extracting $line from public URL" + httpsimple "$line" | tar -xv + ;; + *) + error "Unknown line $line" + ;; + esac +done < .S3file + +exit 0 \ No newline at end of file diff --git a/bin/compile b/bin/compile deleted file mode 100755 index 6856d37..0000000 --- a/bin/compile +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -# bin/compile -# See https://github.com/paulhammond/s3-tarball-buildpack for license and docs - -working=$(pwd) -mkdir -p "$2" -build=$(cd "$1/" && pwd) -cache=$(cd "$2/" && pwd) -env=$(cd "$3/" && pwd) - -# s3simple is a small, simple bash s3 client with minimal dependencies. -# See http://github.com/paulhammond/s3simple for documentation and licence. -s3simple() { - local command="$1" - local url="$2" - local file="${3:--}" - - if [ "${url:0:5}" != "s3://" ]; then - echo "Need an s3 url" - return 1 - fi - local path="${url:4}" - - if [ -z "${AWS_ACCESS_KEY_ID-}" ]; then - echo "Need AWS_ACCESS_KEY_ID to be set" - return 1 - fi - - if [ -z "${AWS_SECRET_ACCESS_KEY-}" ]; then - echo "Need AWS_SECRET_ACCESS_KEY to be set" - return 1 - fi - - local method md5 args - case "$command" in - get) - method="GET" - md5="" - args="-o $file" - ;; - put) - method="PUT" - if [ ! -f "$file" ]; then - echo "file not found" - exit 1 - fi - md5="$(openssl md5 -binary $file | openssl base64)" - args="-T $file -H Content-MD5:$md5" - ;; - *) - echo "Unsupported command" - return 1 - esac - - local date="$(date -u '+%a, %e %b %Y %H:%M:%S +0000')" - local string_to_sign - printf -v string_to_sign "%s\n%s\n\n%s\n%s" "$method" "$md5" "$date" "$path" - local signature=$(echo -n "$string_to_sign" | openssl sha1 -binary -hmac "${AWS_SECRET_ACCESS_KEY}" | openssl base64) - local authorization="AWS ${AWS_ACCESS_KEY_ID}:${signature}" - - curl $args -s -f -H Date:"${date}" -H Authorization:"${authorization}" https://s3.amazonaws.com"${path}" -} - -cd $build - -for e in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do - if [ -f $env/$e ]; then - export "$e=$(cat $env/$e)" - fi -done - -while read line; do - case "$line" in - AWS_*=*) - export "$line" - ;; - s3://*) - echo "-----> s3tarballing $line" - s3simple get "$line" | tar -zx - ;; - *) - echo "unknown line $line" - ;; - esac -done < .buildpack-s3-tarballs diff --git a/bin/detect b/bin/detect index 294714e..35954c0 100755 --- a/bin/detect +++ b/bin/detect @@ -1,10 +1,7 @@ #!/usr/bin/env bash -# bin/detect -# See https://github.com/paulhammond/s3-tarball-buildpack for license and docs -if [ -f $1/.buildpack-s3-tarballs ]; then - echo "S3Tarball" - exit 0 +if [[ -f S3file ]]; then + echo "S3 Tarball" && exit 0 else - exit 1 + echo "no" && exit 1 fi diff --git a/bin/release b/bin/release deleted file mode 100755 index 4ba7583..0000000 --- a/bin/release +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -# bin/release -# See https://github.com/paulhammond/s3-tarball-buildpack for license and docs - -echo "--- {}" diff --git a/buildpack.toml b/buildpack.toml new file mode 100644 index 0000000..3bcfce9 --- /dev/null +++ b/buildpack.toml @@ -0,0 +1,21 @@ +api = "0.5" + +[buildpack] +id = "fagiani/s3-tarball" +version = "0.1.0" +name = "S3 Tarball Cloud Native Buildpack" +homepage = "https://github.com/fagiani/s3-tarball-buildpack" +description = "Downloads and writes files within tarballs from private S3 buckets in build time" +keywords = ["S3", "tar", "tarball", "tgz", "archives", "build", "write", "external", "files"] + +[[buildpack.licenses]] +type = "MIT" + +[[stacks]] +id = "heroku-18" + +[[stacks]] +id = "heroku-20" + +[[stacks]] +id = "io.buildpacks.stacks.bionic" \ No newline at end of file