From 2dcb1fa83c6383eb4383ed6217332ec4f74f5a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Harboe?= Date: Sat, 31 Aug 2024 01:21:44 +0200 Subject: [PATCH] Revert "patcher: Parallelize the read-only patchelf commands" This reverts commit 3e53303a515ee7ae7dc090cf8a146e248987b594. --- MODULE.bazel.lock | 13 ++- patcher.py | 210 +++++++++++----------------------------------- 2 files changed, 63 insertions(+), 160 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 18b4c59..93974ad 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -63,7 +63,7 @@ "moduleExtensions": { "//:extension.bzl%orfs_repositories": { "general": { - "bzlTransitiveDigest": "iKdRL8KmOqQKkYlKI6FA6kzNZLve5/NCiAGHxstf1Wo=", + "bzlTransitiveDigest": "42x9Wez2cJ4mcTzytkWEzBr9ilyB80Y3HGoSJdZwb6w=", "usagesDigest": "vDCy3RMPjTu2/yUpxKaZKZRwXePWohgRn/P6sRnhufM=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -89,6 +89,17 @@ "https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz" ] } + }, + "com_github_docker_buildx_file": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_file", + "attributes": { + "executable": true, + "sha256": "8d486f0088b7407a90ad675525ba4a17d0a537741b9b33fe3391a88cafa2dd0b", + "urls": [ + "https://github.com/docker/buildx/releases/download/v0.15.1/buildx-v0.15.1.linux-amd64" + ] + } } }, "recordedRepoMappingEntries": [ diff --git a/patcher.py b/patcher.py index 707aea8..edb7b4a 100755 --- a/patcher.py +++ b/patcher.py @@ -3,183 +3,75 @@ import argparse import os import subprocess -import multiprocessing -from concurrent.futures import ThreadPoolExecutor -from typing import List, Optional DEFAULT_SEARCH_PATHS = [ - "/lib64", - "/usr/lib/x86_64-linux-gnu", + '/lib64', + '/usr/lib/x86_64-linux-gnu', ] -ELF_MAGIC = b"\x7fELF" +ELF_MAGIC = b'\x7fELF' - -def magic(path: str) -> Optional[bytes]: - """ - Returns first few bytes from the given file. - - Parameters - ---------- - path : str - Path to the file - - Returns - ------- - Optional[bytes] - bytes or None - """ +def magic(path): if not os.path.isfile(path): return None - try: - with open(path, "rb") as f: - return f.read(len(ELF_MAGIC)) - except FileNotFoundError: - return None + with open(path, 'rb') as f: + return f.read(len(ELF_MAGIC)) return None - -def patch_prepare(args: argparse.Namespace, root: str, file: str) -> List: - """ - Reads patchelf information (like rpath or interpreter) - and prepares patchelf commands. It also fixes links. - - Parameters - ---------- - args : argparse.Namespace - Program arguments - root : str - Root directory of the processed file - file : str - Name of the processed file - - Returns - ------- - List - List of prepared commands - - Raises - ------ - subprocess.CalledProcessError - Exception raised when subprocess fails - """ - link = os.path.join(root, file) - if os.path.islink(link) and os.path.isabs(os.readlink(link)): - readlink = os.path.relpath(os.readlink(link), start=os.path.abspath("/")) - target = os.path.join(args.directory, readlink) - link_to_target = os.path.relpath(target, start=root) - os.unlink(link) - os.symlink(link_to_target, link) - return [] - - if magic(os.path.join(root, file)) != ELF_MAGIC: - return [] - - needed_result = subprocess.run( - [args.patchelf, "--print-needed", file], cwd=root, capture_output=True - ) - needed_libs = needed_result.stdout.decode("utf-8").strip() - if not needed_libs: - return [] - - rpath_fragments = ( - subprocess.check_output([args.patchelf, "--print-rpath", file], cwd=root) - .decode("utf-8") - .strip() - ) - rpaths = [] - for rpath in rpath_fragments.split(":") + DEFAULT_SEARCH_PATHS: - if not rpath: - continue - - if "$ORIGIN" in rpath: - rpaths.append(rpath) - else: - elf = os.path.join("/", os.path.relpath(root, start=args.directory)) - elf_to_rpath = os.path.relpath(rpath, start=elf) - rpaths.append(os.path.join("$ORIGIN", elf_to_rpath)) - - rpath = ":".join(rpaths).encode("utf-8") - cmds = [ - ( - [ - args.patchelf, - "--force-rpath", - "--set-rpath", - rpath, - "--no-default-lib", - file, - ], - root, - ) - ] - - interpreter_result = subprocess.run( - [args.patchelf, "--print-interpreter", file], cwd=root, capture_output=True - ) - if interpreter_result.returncode != 0: - return cmds - - interpreter_old = interpreter_result.stdout.decode("utf-8").strip() - execution_root = os.path.normpath(os.path.join(args.directory, "..", "..")) - interp = os.path.relpath(interpreter_old, start="/") - execution_root_to_interp = os.path.relpath( - os.path.join(args.directory, interp), execution_root - ) - cmds.append( - ([args.patchelf, "--set-interpreter", execution_root_to_interp, file], root), - ) - return cmds - - def main(): parser = argparse.ArgumentParser() - parser.add_argument("directory", help="Directory to patch.") - parser.add_argument( - "-p", "--patchelf", default="patchelf", help="`patchelf` binary to use." - ) - parser.add_argument( - "-j", "--jobs", default=None, type=int, help="Number of threads to use." - ) + parser.add_argument('directory', help='Directory to patch.') + parser.add_argument('-p', '--patchelf', default = 'patchelf', help='`patchelf` binary to use.') args = parser.parse_args() - if args.jobs is None: - args.jobs = multiprocessing.cpu_count() // 2 - - futures, commands, failed_files = [], [], [] - with ThreadPoolExecutor(max_workers=args.jobs) as executor: - for root, dirs, files in os.walk(args.directory): - for file in files: - futures.append( - ( - executor.submit( - patch_prepare, - args, - root, - file, - ), - (root, file), - ) - ) - for future, (root, file) in futures: - try: - command = future.result() - except subprocess.CalledProcessError as ex: - failed_files.append((root, file, ex)) + for root, dirs, files in os.walk(args.directory): + for file in files: + link = os.path.join(root, file) + if os.path.islink(link) and os.path.isabs(os.readlink(link)): + target = os.path.join(args.directory, os.readlink(link)) + link_to_target = os.path.relpath(target, start=root) + os.unlink(link) + os.symlink(link_to_target, link) + continue + + if magic(os.path.join(root, file)) != ELF_MAGIC: continue - commands.extend(command) - if failed_files: - error_msg = "\n".join([f"{os.path.join(r, f)}" for r, f, _ in failed_files]) - raise Exception( - f"Cannot prepare patchelf command for:\n{error_msg}" - ) from failed_files[0][2] + needed_result = subprocess.run([args.patchelf, '--print-needed', file], cwd=root, capture_output=True) + needed_libs = needed_result.stdout.decode('utf-8').strip() + if not needed_libs: + continue + + rpath_fragments = subprocess.check_output([args.patchelf, '--print-rpath', file], cwd=root).decode('utf-8').strip() + rpaths = [] + for rpath in rpath_fragments.split(':') + DEFAULT_SEARCH_PATHS: + if not rpath: + continue + + if '$ORIGIN' in rpath: + rpaths.append(rpath) + else: + elf = os.path.join('/', os.path.relpath(root, start=args.directory)) + elf_to_rpath = os.path.relpath(rpath, start=elf) + rpaths.append(os.path.join('$ORIGIN', elf_to_rpath)) + + rpath = ":".join(rpaths).encode('utf-8') + subprocess.check_call([args.patchelf, '--force-rpath', '--set-rpath', rpath, '--no-default-lib', file], cwd=root) + + interpreter_result = subprocess.run([args.patchelf, '--print-interpreter', file], cwd=root, capture_output=True) + if interpreter_result.returncode != 0: + continue + + interpreter_old = interpreter_result.stdout.decode('utf-8').strip() + execution_root = os.path.normpath(os.path.join(args.directory, '..', '..')) + interp = os.path.relpath(interpreter_old, start = '/') + execution_root_to_interp = os.path.relpath(os.path.join(args.directory, interp), execution_root) + subprocess.check_call([args.patchelf, '--set-interpreter', execution_root_to_interp, file], cwd=root) + - for command, root in commands: - subprocess.check_call(command, cwd=root) -if __name__ == "__main__": +if __name__ == '__main__': main()