Skip to content

Commit

Permalink
fix: flip unused_inputs=1 by default (#432)
Browse files Browse the repository at this point in the history
### Changes are visible to end-users: yes/no

- Searched for relevant documentation and updated as needed: yes
- Breaking change (forces users to change their own code or config): no
- Suggested release notes appear below: yes

### Test plan

- New test cases added
  • Loading branch information
thesayyn authored Nov 13, 2024
1 parent 509ed22 commit b9a2c70
Show file tree
Hide file tree
Showing 9 changed files with 2,554 additions and 20 deletions.
6 changes: 5 additions & 1 deletion MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@ module(

# Lower-bound versions of direct dependencies.
# When bumping, add a comment explaining what's required from the newer release.
bazel_dep(name = "aspect_bazel_lib", version = "2.9.1") # py_image_layer requires 2.x for the `tar` rule.

# py_image_layer requires 2.x for the `tar` rule.
# py_image_layer needs compute_unused_inputs attribute
# py_image_layer needs repo_mapping fix.
bazel_dep(name = "aspect_bazel_lib", version = "2.9.4")
bazel_dep(name = "bazel_skylib", version = "1.4.2")
bazel_dep(name = "rules_python", version = "0.29.0")
bazel_dep(name = "platforms", version = "0.0.7")
Expand Down
13 changes: 8 additions & 5 deletions docs/py_image_layer.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions py/private/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ bzl_library(
srcs = ["py_image_layer.bzl"],
deps = [
"@aspect_bazel_lib//lib:tar",
"@aspect_bazel_lib//lib:transitions",
],
)

Expand Down
40 changes: 29 additions & 11 deletions py/private/py_image_layer.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,17 @@ oci_image(
"""

load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar")
load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup")

default_layer_groups = {
# match *only* external pip like repositories that contain the string "site-packages"
"packages": "\\.runfiles/.*/site-packages",
"packages": "\\\\.runfiles/.*/site-packages",
# match *only* external repositories that begins with the string "python"
# e.g. this will match
# `/hello_world/hello_world_bin.runfiles/rules_python~0.21.0~python~python3_9_aarch64-unknown-linux-gnu/bin/python3`
# but not match
# `/hello_world/hello_world_bin.runfiles/_main/python_app`
"interpreter": "\\.runfiles/python.*-.*/",
"interpreter": "\\\\.runfiles/.*python.*-.*/",
}

def _split_mtree_into_layer_groups(name, root, groups, group_names, **kwargs):
Expand Down Expand Up @@ -90,7 +91,7 @@ awk < $< 'BEGIN {
)


def py_image_layer(name, py_binary, root = None, layer_groups = {}, compress = "gzip", tar_args = ["--options", "gzip:!timestamp"], **kwargs):
def py_image_layer(name, binary, root = "/", layer_groups = {}, compress = "gzip", tar_args = ["--options", "gzip:!timestamp"], compute_unused_inputs = 1, platform = None, **kwargs):
"""Produce a separate tar output for each layer of a python app
> Requires `awk` to be installed on the host machine/rbe runner.
Expand All @@ -113,11 +114,13 @@ def py_image_layer(name, py_binary, root = None, layer_groups = {}, compress = "
Args:
name: base name for targets
py_binary: a py_binary target
binary: a py_binary target
root: Path to where the layers should be rooted. If not specified, the layers will be rooted at the workspace root.
layer_groups: Additional layer groups to create. They are used to group files into layers based on their path. In the form of: ```{"<name>": "regex_to_match_against_file_paths"}```
compress: Compression algorithm to use. Default is gzip. See: https://github.com/bazel-contrib/bazel-lib/blob/main/docs/tar.md#tar_rule
tar_args: Additional arguments to pass to the tar rule. Default is `["--options", "gzip:!timestamp"]`. See: https://github.com/bazel-contrib/bazel-lib/blob/main/docs/tar.md#tar_rule
compress: Compression algorithm to use. Default is gzip. See: https://github.com/bazel-contrib/bazel-lib/blob/main/docs/tar.md#tar_rule-compress
compute_unused_inputs: Whether to compute unused inputs. Default is 1. See: https://github.com/bazel-contrib/bazel-lib/blob/main/docs/tar.md#tar_rule-compute_unused_inputs
platform: The platform to use for the transition. Default is None. See: https://github.com/bazel-contrib/bazel-lib/blob/main/docs/transitions.md#platform_transition_binary-target_platform
tar_args: Additional arguments to pass to the tar rule. Default is `["--options", "gzip:!timestamp"]`. See: https://github.com/bazel-contrib/bazel-lib/blob/main/docs/tar.md#tar_rule-args
**kwargs: attribute that apply to all targets expanded by the macro
Returns:
Expand All @@ -130,7 +133,7 @@ def py_image_layer(name, py_binary, root = None, layer_groups = {}, compress = "
# into fine-grained layers for better pull, push and remote cache performance.
mtree_spec(
name = name + ".manifest",
srcs = [py_binary],
srcs = [binary],
**kwargs
)

Expand All @@ -141,17 +144,32 @@ def py_image_layer(name, py_binary, root = None, layer_groups = {}, compress = "
_split_mtree_into_layer_groups(name, root, groups, group_names, **kwargs)

# Finally create layers using the tar rule
result = []
srcs = []
for group_name in group_names:
tar_target = "_{}_{}".format(name, group_name)
tar(
name = tar_target,
srcs = [py_binary],
srcs = [binary],
mtree = "{}.{}.manifest.spec".format(name, group_name),
compress = compress,
compute_unused_inputs = compute_unused_inputs,
args = tar_args,
**kwargs
)
result.append(tar_target)
srcs.append(tar_target)

if platform:
platform_transition_filegroup(
name = name,
srcs = srcs,
target_platform = platform,
**kwargs
)
else:
native.filegroup(
name = name,
srcs = srcs,
**kwargs
)

return result
return srcs
6 changes: 3 additions & 3 deletions py/repositories.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ def rules_py_dependencies():
# py_image_layer requires 2.x for the `tar` rule.
http_archive(
name = "aspect_bazel_lib",
sha256 = "f93d386d8d0b0149031175e81df42a488be4267c3ca2249ba5321c23c60bc1f0",
strip_prefix = "bazel-lib-2.9.1",
url = "https://github.com/bazel-contrib/bazel-lib/releases/download/v2.9.1/bazel-lib-v2.9.1.tar.gz",
sha256 = "349aabd3c2b96caeda6181eb0ae1f14f2a1d9f3cd3c8b05d57f709ceb12e9fb3",
strip_prefix = "bazel-lib-2.9.4",
url = "https://github.com/bazel-contrib/bazel-lib/releases/download/v2.9.4/bazel-lib-v2.9.4.tar.gz",
)

http_archive(
Expand Down
29 changes: 29 additions & 0 deletions py/tests/py_image_layer/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
load("//py:defs.bzl", "py_image_layer", "py_binary")
load("asserts.bzl", "assert_tar_listing")

platform(
name = "linux_amd64",
constraint_values = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
],
)


# Case 1: Basic usage
py_binary(
name = "my_app_bin",
srcs = ["main.py"],
)

py_image_layer(
name = "my_app_layers",
binary = ":my_app_bin",
platform = ":linux_amd64",
)

assert_tar_listing(
name = "my_app_layers_test",
actual = [":my_app_layers"],
expected = ":my_app_layers.listing",
)
21 changes: 21 additions & 0 deletions py/tests/py_image_layer/asserts.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_file")

# buildifier: disable=function-docstring
def assert_tar_listing(name, actual, expected):
actual_listing = "_{}_listing".format(name)
native.genrule(
name = actual_listing,
srcs = actual,
testonly = True,
outs = ["_{}.listing".format(name)],
cmd = 'echo $(SRCS) | TZ="UTC" LC_ALL="en_US.UTF-8" xargs -n 1 $(BSDTAR_BIN) --exclude "*/_repo_mapping" --exclude "**/tools/venv_bin/**" -tvf > $@'.format(actual),
toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"],
)

write_source_file(
name = name,
in_file = actual_listing,
out_file = expected,
testonly = True,
tags = ["skip-on-bazel6"],
)
1 change: 1 addition & 0 deletions py/tests/py_image_layer/main.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
print("Hello!!")
Loading

0 comments on commit b9a2c70

Please sign in to comment.