diff --git a/HaikuPorter/BuildMaster.py b/HaikuPorter/BuildMaster.py index edeb7c8a..d07a21c0 100644 --- a/HaikuPorter/BuildMaster.py +++ b/HaikuPorter/BuildMaster.py @@ -43,18 +43,18 @@ def filter(self, record): class ScheduledBuild(object): - def __init__(self, port, portsTreePath, requiredPackageIDs, packagesPath, - presentDependencyPackages): + def __init__(self, port, portsTreePath, missingPackageIDs, + packageRepository, presentDependencyPackages): self.port = port self.recipeFilePath \ = os.path.relpath(port.recipeFilePath, portsTreePath) self.resultingPackages \ = [package.hpkgName for package in self.port.packages] - self.packagesPath = packagesPath + self.packageRepository = packageRepository self.requiredPackages = presentDependencyPackages self.requiredPackageIDs = [ os.path.basename(path) for path in presentDependencyPackages] - self.missingPackageIDs = set(requiredPackageIDs) + self.missingPackageIDs = set(missingPackageIDs) self.buildNumbers = [] self.lost = False @@ -69,7 +69,7 @@ def packageCompleted(self, package, available): self.missingPackageIDs.remove(packageID) self.requiredPackageIDs.append(package.hpkgName) self.requiredPackages.append( - os.path.join(self.packagesPath, package.hpkgName)) + self.packageRepository.packagePath(package.hpkgName)) else: self.lost = True @@ -151,7 +151,7 @@ def status(self): class BuildMaster(object): - def __init__(self, portsTreePath, packagesPath, options): + def __init__(self, portsTreePath, packageRepository, options): self.portsTreePath = portsTreePath self._fillPortsTreeInfo() @@ -159,7 +159,7 @@ def __init__(self, portsTreePath, packagesPath, options): self.reconnectingBuilders = [] self.lostBuilders = [] self.availableBuilders = [] - self.packagesPath = packagesPath + self.packageRepository = packageRepository self.masterBaseDir = os.path.realpath('buildmaster') self.builderBaseDir = os.path.join(self.masterBaseDir, 'builders') self.buildOutputBaseDir = getOption('buildMasterOutputDir') @@ -230,9 +230,9 @@ def __init__(self, portsTreePath, packagesPath, options): builder = None try: - builder = RemoteBuilderSSH(configFilePath, packagesPath, - self.buildOutputBaseDir, self.portsTreeOriginURL, - self.portsTreeHead) + builder = RemoteBuilderSSH(configFilePath, + packageRepository, self.buildOutputBaseDir, + self.portsTreeOriginURL, self.portsTreeHead) except Exception as exception: self.logger.error('failed to add builder from config ' + configFilePath + ':' + str(exception)) @@ -247,7 +247,7 @@ def __init__(self, portsTreePath, packagesPath, options): for i in range(0, self.localBuilders): builder = None try: - builder = LocalBuilder(str(i), packagesPath, + builder = LocalBuilder(str(i), packageRepository, self.buildOutputBaseDir, options) except Exception as exception: self.logger.error('failed to add local builder: ' @@ -292,20 +292,20 @@ def addSkipped(self, port, reason): self.skippedBuilds.append(skippedBuild) self._reportStatus() - def schedule(self, port, requiredPackageIDs, presentDependencyPackages): + def schedule(self, port, missingPackageIDs, presentDependencyPackages): # Skip builds that would overwrite existing packages. for package in port.packages: - packagePath = os.path.join(self.packagesPath, package.hpkgName) - if not os.path.exists(packagePath): + if not self.packageRepository.hasPackage(package.hpkgName): continue - self.addSkipped(port, 'some packages already exist at ' - + self.packagesPath + ', revision bump required') + self.addSkipped(port, 'some packages already exist in package' + + ' repository, revision bump required') return self.logger.info('scheduling build of ' + port.versionedName) scheduledBuild = ScheduledBuild(port, self.portsTreePath, - requiredPackageIDs, self.packagesPath, presentDependencyPackages) + missingPackageIDs, self.packageRepository, + presentDependencyPackages) if scheduledBuild.buildable: self.scheduledBuilds.append(scheduledBuild) diff --git a/HaikuPorter/Builders/LocalBuilder.py b/HaikuPorter/Builders/LocalBuilder.py index 8985816e..a2b23d67 100644 --- a/HaikuPorter/Builders/LocalBuilder.py +++ b/HaikuPorter/Builders/LocalBuilder.py @@ -14,12 +14,12 @@ class LocalBuilder(object): - def __init__(self, name, packagesPath, outputBaseDir, options): + def __init__(self, name, packageRepository, outputBaseDir, options): self.options = options self.name = name self.buildCount = 0 self.failedBuilds = 0 - self.packagesPath = packagesPath + self.packageRepository = packageRepository self.state = BuilderState.AVAILABLE self.currentBuild = None diff --git a/HaikuPorter/Builders/RemoteBuilderSSH.py b/HaikuPorter/Builders/RemoteBuilderSSH.py index e8637178..1d32de9b 100644 --- a/HaikuPorter/Builders/RemoteBuilderSSH.py +++ b/HaikuPorter/Builders/RemoteBuilderSSH.py @@ -15,6 +15,7 @@ # These usages kinda need refactored from ..ConfigParser import ConfigParser from ..Configuration import Configuration +from ..Options import getOption from .Builder import BuilderState try: @@ -23,14 +24,14 @@ paramiko = None class RemoteBuilderSSH(object): - def __init__(self, configFilePath, packagesPath, outputBaseDir, + def __init__(self, configFilePath, packageRepository, outputBaseDir, portsTreeOriginURL, portsTreeHead): self._loadConfig(configFilePath) self.availablePackages = [] self.visiblePackages = [] self.portsTreeOriginURL = portsTreeOriginURL self.portsTreeHead = portsTreeHead - self.packagesPath = packagesPath + self.packageRepository = packageRepository if not paramiko: raise Exception('paramiko unavailable') @@ -231,13 +232,20 @@ def _getAvailablePackages(self): def _removeObsoletePackages(self): cachePath = self.config['portstree']['packagesCachePath'] + systemPackagesDirectory = getOption('systemPackagesDirectory') + for entry in list(self.availablePackages): - if not os.path.exists(os.path.join(self.packagesPath, entry)): - self.logger.info( - 'removing obsolete package {} from cache'.format(entry)) - entryPath = cachePath + '/' + entry - self.sftpClient.remove(entryPath) - self.availablePackages.remove(entry) + if self.packageRepository.hasPackage(entry): + continue + + if os.path.exists(os.path.join(systemPackagesDirectory, entry)): + continue + + self.logger.info( + 'removing obsolete package {} from cache'.format(entry)) + entryPath = cachePath + '/' + entry + self.sftpClient.remove(entryPath) + self.availablePackages.remove(entry) def _setupForBuilding(self): if self.state == BuilderState.AVAILABLE: @@ -331,11 +339,11 @@ def runBuild(self): self.buildLogger.info('download package ' + package.hpkgName + ' from builder') - packageFile = os.path.join(self.packagesPath, package.hpkgName) - downloadFile = packageFile + '.download' - self._getFile(self.config['portstree']['packagesPath'] + '/' - + package.hpkgName, downloadFile) - os.rename(downloadFile, packageFile) + remotePath = self.config['portstree']['packagesPath'] + '/' \ + + package.hpkgName + with self.sftpClient.file(remotePath, 'r') as remoteFile: + self.packageRepository.writePackage(package.hpkgName, + remoteFile) self._purgePort(scheduledBuild) self._clearVisiblePackages() @@ -374,12 +382,7 @@ def _symlink(self, sourcePath, destPath): self.sftpClient.symlink(sourcePath, destPath) def _move(self, sourcePath, destPath): - # Unfortunately we can't use SFTPClient.rename as that uses the rename - # command (vs. posix-rename) which uses hardlinks which fail on BFS - (output, channel) = self._remoteCommand('mv "' + sourcePath + '" "' - + destPath + '"') - if channel.recv_exit_status() != 0: - raise IOError('failed moving {} to {}'.format(sourcePath, destPath)) + self.sftpClient.posix_rename(sourcePath, destPath) def _openRemoteFile(self, path, mode): return self.sftpClient.open(path, mode) @@ -430,7 +433,9 @@ def _makePackageAvailable(self, packagePath): = self.config['portstree']['packagesCachePath'] + '/' + packageName uploadPath = entryPath + '.upload' - self._putFile(packagePath, uploadPath) + with self.sftpClient.file(uploadPath, 'w') as remoteFile: + self.packageRepository.readPackage(packagePath, remoteFile) + self._move(uploadPath, entryPath) self.availablePackages.append(packageName) diff --git a/HaikuPorter/Main.py b/HaikuPorter/Main.py index 2383c7ed..5a583e97 100644 --- a/HaikuPorter/Main.py +++ b/HaikuPorter/Main.py @@ -109,7 +109,11 @@ def run(self, args): if self.options.buildMaster: from .BuildMaster import BuildMaster - self.buildMaster = BuildMaster(self.treePath, self.packagesPath, + + packageRepository = PackageRepository(self.packagesPath, + None, self.options.quiet, self.options.verbose) + + self.buildMaster = BuildMaster(self.treePath, packageRepository, self.options) self.options.allDependencies = True diff --git a/HaikuPorter/Options.py b/HaikuPorter/Options.py index bf0889a6..e648390c 100644 --- a/HaikuPorter/Options.py +++ b/HaikuPorter/Options.py @@ -305,6 +305,9 @@ def parseOptions(): advanced_flags.add_option('--sign-package-repository-privkey-pass', action='store', type='string', dest='packageRepositorySignPrivateKeyPass', default=None, help='sign the package repository with the given minisign password') + advanced_flags.add_option('--storage-backend-config', action='store', + dest='storageBackendConfig', default=None, type='string', + help='use the given file as the storage backend config'), advanced_flags.add_option('--check-package-repository-consistency', action='store_true', dest='checkPackageRepositoryConsistency', default=False, help='check consistency of package repository by' diff --git a/HaikuPorter/Package.py b/HaikuPorter/Package.py index 8048c850..986b7db3 100644 --- a/HaikuPorter/Package.py +++ b/HaikuPorter/Package.py @@ -180,18 +180,6 @@ def removeDependencyInfoFromRepository(self, repositoryPath): if os.path.exists(dependencyInfoFile): os.remove(dependencyInfoFile) - def obsoletePackage(self, packagesPath): - """Moves the package-file into the 'obsolete' sub-directory""" - - obsoleteDir = packagesPath + '/.obsolete' - packageFile = packagesPath + '/' + self.hpkgName - if os.path.exists(packageFile): - print('\tobsoleting package ' + self.hpkgName) - obsoletePackage = obsoleteDir + '/' + self.hpkgName - if not os.path.exists(obsoleteDir): - os.mkdir(obsoleteDir) - os.rename(packageFile, obsoletePackage) - def generateDependencyInfoWithoutProvides(self, dependencyInfoPath, requiresToUse): """Create a .DependencyInfo file that doesn't include any provides diff --git a/HaikuPorter/PackageInfo.py b/HaikuPorter/PackageInfo.py index 6d708f62..2a4bc6b2 100644 --- a/HaikuPorter/PackageInfo.py +++ b/HaikuPorter/PackageInfo.py @@ -96,7 +96,8 @@ def _initializeCache(cls): entry = pickle.load(cacheFile) path = entry['path'] if not os.path.exists(path) \ - or os.path.getmtime(path) > entry['modifiedTime']: + or (os.path.getsize(path) != 0 \ + and os.path.getmtime(path) > entry['modifiedTime']): prune = True continue diff --git a/HaikuPorter/PackageRepository.py b/HaikuPorter/PackageRepository.py index 4eb9de7a..79cbd8e6 100644 --- a/HaikuPorter/PackageRepository.py +++ b/HaikuPorter/PackageRepository.py @@ -7,7 +7,9 @@ import glob import hashlib +import json import os +import shutil import subprocess from .Configuration import Configuration @@ -36,11 +38,50 @@ def __init__(self, packagesPath, repository, quiet, verbose): self.quiet = quiet self.verbose = verbose + self._storageBackendInitialized = False + self._storageBackend = None + def prune(self): self.obsoletePackagesWithoutPort() self.obsoletePackagesNewerThanActiveVersion() self.obsoletePackagesWithNewerVersions() + def packageName(self, packagePath): + return os.path.basename(packagePath) + + def packagePath(self, packageName): + return os.path.join(self.packagesPath, packageName) + + def hasPackage(self, packageName): + return os.path.exists(self.packagePath(packageName)) + + def isPackageLocal(self, packagePath): + if not os.path.exists(packagePath): + return False + + packageStat = os.stat(packagePath) + return packageStat.st_size != 0 + + @property + def storageBackend(self): + if not self._storageBackendInitialized: + configFilePath = getOption('storageBackendConfig') + if configFilePath: + with open(configFilePath, 'r') as configFile: + config = json.loads(configFile.read()) + + backendType = config.get('backend_type') + if backendType == 's3': + from .StorageBackendS3 import StorageBackendS3 + self._storageBackend = StorageBackendS3(self.packagesPath, + config) + else: + raise Exception(f'unknown backend type {backendType}') + + self._storageBackendInitialized = True + + return self._storageBackend + def packageList(self, packageSpec=None): if packageSpec is None: packageSpec = '' @@ -50,6 +91,27 @@ def packageList(self, packageSpec=None): packageSpec += '*.hpkg' return glob.glob(os.path.join(self.packagesPath, packageSpec)) + def readPackage(self, packagePath, file): + if self.isPackageLocal(packagePath): + with open(packagePath, 'rb') as packageFile: + shutil.copyfileobj(packageFile, file) + return + + packageName = self.packageName(packagePath) + if self.storageBackend is not None: + self.storageBackend.readPackage(packageName, file) + return + + raise Exception(f'package {packageName} unavailable') + + def writePackage(self, packageName, file): + packagePath = self.packagePath(packageName) + temporaryPath = packagePath + '.temp' + with open(temporaryPath, 'wb') as packageFile: + shutil.copyfileobj(file, packageFile) + + os.rename(temporaryPath, packagePath) + def packageInfoList(self, packageSpec=None): result = [] for package in self.packageList(packageSpec): @@ -67,7 +129,7 @@ def packageInfoList(self, packageSpec=None): return result def obsoletePackage(self, path, reason=None): - packageFileName = os.path.basename(path) + packageFileName = self.packageName(path) if not self.quiet: print('\tobsoleting package {}: {}'.format(packageFileName, reason)) @@ -136,16 +198,22 @@ def createPackageRepository(self, outputPath): for package in glob.glob(os.path.join(repoPackagesPath, '*.hpkg')): os.unlink(package) + localPackages = [] packageList = self.packageInfoList() for package in packageList: + if not self.isPackageLocal(package.path): + continue + os.link(package.path, - os.path.join(repoPackagesPath, os.path.basename(package.path))) + os.path.join(repoPackagesPath, self.packageName(package.path))) + + localPackages.append(package.path) packageListFile = os.path.join(outputPath, 'package.list') + packageNameList \ + = [self.packageName(package.path) for package in packageList] with open(packageListFile, 'w') as outputFile: - fileList = '\n'.join( - [os.path.basename(package.path) for package in packageList]) - outputFile.write(fileList) + outputFile.write('\n'.join(packageNameList)) if not os.path.exists(repoFile): if not packageList: @@ -156,14 +224,25 @@ def createPackageRepository(self, outputPath): stderr=subprocess.STDOUT).decode('utf-8') info(output) + if self.storageBackend is not None: + self._populateStorageBackendPackages(localPackages) + output = subprocess.check_output([packageRepoCommand, 'update', '-C', - repoPackagesPath, '-v', repoFile, repoFile, packageListFile], - stderr=subprocess.STDOUT).decode('utf-8') + repoPackagesPath, '-v', '-t', repoFile, repoFile, + packageListFile], stderr=subprocess.STDOUT).decode('utf-8') info(output) - self._checksumPackageRepository(repoFile) + + repoChecksumFile = repoFile + '.sha256' + self._checksumPackageRepository(repoFile, repoChecksumFile) self._signPackageRepository(repoFile) - def _checksumPackageRepository(self, repoFile): + if self.storageBackend is not None: + self._stubLocalPackages(localPackages) + self._populateStorageBackendFiles( + [repoInfoFile, repoFile, repoChecksumFile, packageListFile]) + self._pruneStorageBackend(packageNameList) + + def _checksumPackageRepository(self, repoFile, repoChecksumFile): """Create a checksum of the package repository""" checksum = hashlib.sha256() with open(repoFile, 'rb') as inputFile: @@ -172,7 +251,8 @@ def _checksumPackageRepository(self, repoFile): if not data: break checksum.update(data) - with open(repoFile + '.sha256', 'w') as outputFile: + + with open(repoChecksumFile, 'w') as outputFile: outputFile.write(checksum.hexdigest()) def _signPackageRepository(self, repoFile): @@ -191,7 +271,7 @@ def _signPackageRepository(self, repoFile): minisignCommand = Configuration.getMinisignCommand() if not minisignCommand: sysExit('minisign command missing to sign repository!') - + # minisign -s /tmp/minisign.key -Sm ${ARTIFACT} info("signing repository") output = subprocess.check_output([minisignCommand, '-s', @@ -220,3 +300,31 @@ def checkPackageRepositoryConsistency(self): except LookupError as error: print('{}:\n{}\n'.format(os.path.relpath(package.path, self.packagesPath), prefixLines('\t', str(error)))) + + def _populateStorageBackendPackages(self, localPackages): + for packagePath in localPackages: + packageName = self.packageName(packagePath) + info(f'uploading package {packageName} to storage backend') + + with open(packagePath, 'rb') as packageFile: + self.storageBackend.writePackage(packageName, packageFile) + + def _populateStorageBackendFiles(self, files): + for filePath in files: + fileName = os.path.basename(filePath) + info(f'uploading {fileName} to storage backend') + + with open(filePath, 'rb') as inputFile: + self.storageBackend.writeFile(fileName, inputFile) + + def _stubLocalPackages(self, localPackages): + for packagePath in localPackages: + os.truncate(packagePath, 0) + + def _pruneStorageBackend(self, packageNameList): + for remotePackage in self.storageBackend.listPackages(): + if remotePackage in packageNameList: + continue + + info(f'delete package {remotePackage} from storage backend') + self.storageBackend.deletePackage(remotePackage) diff --git a/HaikuPorter/Port.py b/HaikuPorter/Port.py index 0dd1454e..4d30c5b5 100644 --- a/HaikuPorter/Port.py +++ b/HaikuPorter/Port.py @@ -497,13 +497,6 @@ def removeDependencyInfosFromRepository(self): for package in self.packages: package.removeDependencyInfoFromRepository(self._repositoryDir) - def obsoletePackages(self, packagesPath): - """Moves all package-files into the 'obsolete' sub-directory""" - - self.parseRecipeFileIfNeeded() - for package in self.packages: - package.obsoletePackage(packagesPath) - @property def mainPackage(self): self.parseRecipeFileIfNeeded() diff --git a/HaikuPorter/StorageBackendS3.py b/HaikuPorter/StorageBackendS3.py new file mode 100644 index 00000000..9d5a1d26 --- /dev/null +++ b/HaikuPorter/StorageBackendS3.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2024 Michael Lotz +# Distributed under the terms of the MIT License. + +# -- Modules ------------------------------------------------------------------ + +import boto3 + +from contextlib import contextmanager + +# -- StorageBackendS3 class --------------------------------------------------- + +class StorageBackendS3(): + def __init__(self, packagesPath, config): + if 'endpoint_url' not in config: + raise Exception('missing endpoint_url in s3 config') + if 'access_key_id' not in config: + raise Exception('missing access_key_id in s3 config') + if 'secret_access_key' not in config: + raise Exception('missing secret_access_key in s3 config') + if 'bucket_name' not in config: + raise Exception('missing bucket_name in s3 config') + + self.bucketName = config['bucket_name'] + self.prefix = config.get('prefix', '') + self.packagesPrefix = self.prefix + 'packages/' + + self.client = boto3.client('s3', + endpoint_url=config['endpoint_url'], + aws_access_key_id=config['access_key_id'], + aws_secret_access_key=config['secret_access_key']) + + def readPackage(self, packageName, file): + self.client.download_fileobj(self.bucketName, + f'{self.packagesPrefix}{packageName}', file) + + def writePackage(self, packageName, file): + self.client.upload_fileobj(file, self.bucketName, + f'{self.packagesPrefix}{packageName}') + + def writeFile(self, fileName, file): + self.client.upload_fileobj(file, self.bucketName, + f'{self.prefix}{fileName}') + + def listPackages(self): + kwargs = { + 'Bucket': self.bucketName, + 'Prefix': self.packagesPrefix + } + + result = [] + while True: + response = self.client.list_objects_v2(**kwargs) + contents = response['Contents'] + for item in contents: + result.append(item['Key'].removeprefix(self.packagesPrefix)) + + if not response.get('IsTruncated', False): + break + + kwargs['StartAfter'] = contents[-1]['Key'] + + return result + + def deletePackage(self, packageName): + self.client.delete_object(Bucket=self.bucketName, + Key=f'{self.packagesPrefix}{packageName}') diff --git a/buildmaster/backend/Dockerfile b/buildmaster/backend/Dockerfile index 4e4ac55d..4caf1a61 100644 --- a/buildmaster/backend/Dockerfile +++ b/buildmaster/backend/Dockerfile @@ -17,30 +17,33 @@ FROM debian:bullseye-slim # hardlink for build-packages ADD https://cgit.haiku-os.org/haiku/plain/src/tools/hardlink_packages.py /usr/local/bin/ -# Haikuporter from local context root (this is where the weird context requirement comes from) -ADD . /tmp/haikuporter - # Pre-requirements RUN apt-get update \ && apt-get -y install attr autoconf automake bison coreutils curl flex \ gawk gcc gcc-multilib g++ git libcurl4-openssl-dev make nasm python3 \ python3-paramiko python3-pip tar texinfo vim wget zlib1g-dev \ - && apt-get clean \ - && echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib' >> /etc/bash.bashrc \ + && apt-get clean + +RUN echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib' >> /etc/bash.bashrc \ && wget https://github.com/jedisct1/minisign/releases/download/0.10/minisign-0.10-linux.tar.gz -O /tmp/minisign.tar.gz \ - && cd /tmp && tar -xvz --strip=2 -f /tmp/minisign.tar.gz && mv minisign /usr/local/bin \ - && pip3 install /tmp/haikuporter \ + && cd /tmp && tar -xvz --strip=2 -f /tmp/minisign.tar.gz && mv minisign /usr/local/bin + +# Haikuporter from local context root (this is where the weird context requirement comes from) +ADD . /tmp/haikuporter + +RUN pip3 install /tmp/haikuporter \ && echo "Bug #277 Fix" && cp /tmp/haikuporter/haikuporter.py /usr/local/bin/haikuporter \ && cp /tmp/haikuporter/buildmaster/backend/assets/bin/* /usr/local/bin/ \ && cp /tmp/haikuporter/buildmaster/backend/assets/bootstrap /bin/ \ && cp /tmp/haikuporter/buildmaster/backend/assets/loop /bin/ \ && rm -rf /tmp/* \ - && mkdir /var/sources /var/packages /var/buildmaster \ + && mkdir /var/licenses /var/buildmaster \ && chmod 755 /usr/local/bin/* COPY --from=host-tools /tmp/haiku/generated/objects/linux/x86_64/release/tools/package/package /usr/local/bin/ COPY --from=host-tools /tmp/haiku/generated/objects/linux/x86_64/release/tools/package_repo/package_repo /usr/local/bin/ COPY --from=host-tools /tmp/haiku/generated/objects/linux/lib/* /usr/local/lib/ +COPY --from=host-tools /tmp/haiku/data/system/data/licenses /var/licenses -VOLUME ["/var/sources", "/var/packages", "/var/buildmaster"] +VOLUME ["/var/buildmaster"] WORKDIR /var/buildmaster diff --git a/buildmaster/backend/README.md b/buildmaster/backend/README.md index 33d9aaa6..12ad5fef 100644 --- a/buildmaster/backend/README.md +++ b/buildmaster/backend/README.md @@ -12,13 +12,15 @@ One buildmaster container per architecture ## Environmental * ```BUILD_TARGET_ARCH``` - Target architecture for buildmaster + * ```SYSTEM_PACKAGE_BRANCH``` - The branch of the system packages + * system-packages are expected at /var/buildmaster/system-packages/$SYSTEM_PACKAGE_BRANCH + * ```STORAGE_BACKEND_CONFIG``` - The path of an external storage backend config file (optional) + * This would normally be pointed at a secret * ```REPOSITORY_TRIGGER_URL``` - Target URL to hit when build complete (optional) * example: https://depot.haiku-os.org/__repository/haikuports/source/haikuports_x86_64/import ## Volumes - * /var/sources (shared between all architectures) - * Storage for various required sources like haiku * /var/packages (shared between all architectures) * Storage for packages and repositories * repository diff --git a/buildmaster/backend/assets/bin/buildmaster b/buildmaster/backend/assets/bin/buildmaster index f0d9d58b..769183fd 100755 --- a/buildmaster/backend/assets/bin/buildmaster +++ b/buildmaster/backend/assets/bin/buildmaster @@ -27,7 +27,6 @@ export SYSTEM_PACKAGES_DIR="$BASE_DIR/system-packages/$SYSTEM_PACKAGE_BRANCH" export HAIKUPORTS_DIR="$BASE_DIR/haikuports" export WORKING_DIR="$HAIKUPORTS_DIR/buildmaster" export OUTPUT_DIR="$BASE_DIR/output" -export REPO_DIR="/var/packages/repository/$BUILD_TARGET_BRANCH/$BUILD_TARGET_ARCH/current" export BUILDRUN_BASE="$OUTPUT_DIR/buildruns" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib" @@ -123,6 +122,7 @@ ln -rs "$BUILDRUN_OUTPUT_DIR" "$BUILDRUN_BASE/current" haikuporter --debug --build-master-output-dir="$BUILDRUN_OUTPUT_DIR" \ --system-packages-directory="$SYSTEM_PACKAGES_DIR" \ + --storage-backend-config="$STORAGE_BACKEND_CONFIG" \ --build-master $PORTS_TO_BUILD BUILDMASTER_RESULT=$? diff --git a/buildmaster/backend/assets/bin/createbuilder b/buildmaster/backend/assets/bin/createbuilder index 4b4e8c38..4f4a4f32 100755 --- a/buildmaster/backend/assets/bin/createbuilder +++ b/buildmaster/backend/assets/bin/createbuilder @@ -31,10 +31,10 @@ if not os.path.isdir(args.confdir): sys.exit() if not os.path.isdir(args.confdir): - os.mkdir(args.confdir, 0755) + os.mkdir(args.confdir, 0o755) if not os.path.isdir(args.confdir + "/keys"): - os.mkdir(args.confdir + "/keys", 0700) + os.mkdir(args.confdir + "/keys", 0o700) builder_json_path = args.confdir + "/" + args.name + ".json" private_key_path = args.confdir + "/keys/" + args.name diff --git a/buildmaster/backend/assets/bin/update_build_packages b/buildmaster/backend/assets/bin/update_build_packages index 48dd485a..c01402d6 100755 --- a/buildmaster/backend/assets/bin/update_build_packages +++ b/buildmaster/backend/assets/bin/update_build_packages @@ -14,7 +14,7 @@ if [ $# -ne 1 ]; then echo " 1. Please ensure latest build-packages have been placed in $BASE_DIR" echo " 2. Note that the [jam RemotePackageRepository file] will be modified." echo " After this tool modifies it, you must check it into git as-is without" - echo " modification of any kind. (the repo is based on the sha256 of it)" + echo " modification of any kind. (the repo is based on the sha256 of it)" echo "" exit 1 fi diff --git a/buildmaster/backend/assets/bootstrap b/buildmaster/backend/assets/bootstrap index a0b47600..193f2989 100755 --- a/buildmaster/backend/assets/bootstrap +++ b/buildmaster/backend/assets/bootstrap @@ -1,7 +1,7 @@ #!/bin/sh BASE_DIR="/var/buildmaster" -SOURCE_DIR="/var/sources" +LICENSES_DIR="/var/licenses" if [ -z "$1" ] then @@ -41,12 +41,10 @@ do shift done -BUILDTOOLS_DIR=$SOURCE_DIR/buildtools -HAIKU_DIR=$SOURCE_DIR/haiku -GENERATED_DIR=$BASE_DIR/generated PORTS_DIR=$BASE_DIR/haikuports OUTPUT_DIR=$BASE_DIR/output -BUILDERS_DIR=$PORTS_DIR/buildmaster/builders +STATE_DIR=$PORTS_DIR/buildmaster +BUILDERS_DIR=$STATE_DIR/builders if [ -z "$ARCH" ] then @@ -73,22 +71,13 @@ else echo "Using existing HaikuPorts repository at $PORTS_DIR" fi -### Get Haiku - -if [ ! -d "$HAIKU_DIR" ]; then - echo "Cloning Haiku repository to $HAIKU_DIR" - git clone --depth=1 https://review.haiku-os.org/haiku "$HAIKU_DIR" -else - echo "Using existing Haiku repository at $HAIKU_DIR" -fi - # Configure the ports tree. cd "$PORTS_DIR" echo "Configuring ports tree" echo "TREE_PATH=\"$PORTS_DIR\"" > haikuports.conf -echo "LICENSES_DIRECTORY=\"$HAIKU_DIR/data/system/data/licenses\"" \ +echo "LICENSES_DIRECTORY=\"$LICENSES_DIR\"" \ >> haikuports.conf echo "PACKAGE_COMMAND=\"package\"" >> haikuports.conf echo "PACKAGE_REPO_COMMAND=\"package_repo\"" >> haikuports.conf @@ -103,8 +92,12 @@ fi # Create some buildmaster paths mkdir -p $OUTPUT_DIR +mkdir -p $STATE_DIR mkdir -p $BUILDERS_DIR +# Record initial processed revision +git rev-parse HEAD > $STATE_DIR/processed_rev + # Done. echo "" diff --git a/buildmaster/backend/assets/loop b/buildmaster/backend/assets/loop index 7830c59e..c4e7cef2 100755 --- a/buildmaster/backend/assets/loop +++ b/buildmaster/backend/assets/loop @@ -2,6 +2,7 @@ BASE_DIR="/var/buildmaster" WORKDIR="$BASE_DIR/haikuports" +OUTPUT_DIR="$BASE_DIR/output" if [ ! -d "$WORKDIR" ] then @@ -22,14 +23,14 @@ ERROR_WAIT=60 export PYTHONUNBUFFERED=1 export BUILD_TARGET_ARCH="$BUILD_TARGET_ARCH" -export REPO_DIR="/var/packages/repository/master/$BUILD_TARGET_ARCH/current" +export REPO_DIR="$BASE_DIR/repository" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib" export SYSTEM_PACKAGES_DIR="$BASE_DIR/system-packages/$SYSTEM_PACKAGE_BRANCH" exec 2>&1 build_prep () { - CONSISTENCY_REPORT_FILE="$REPO_DIR/repo_consistency.txt" + CONSISTENCY_REPORT_FILE="$OUTPUT_DIR/repo_consistency.txt" echo "repo consistency report at $(git rev-parse HEAD)" \ > "$CONSISTENCY_REPORT_FILE" haikuporter --debug --check-repository-consistency \ @@ -43,7 +44,7 @@ build_prep () { while true do if [ ! -d $SYSTEM_PACKAGES_DIR ]; then - echo "system-packages missing at $SYSTEM_PACKAGES_DIRECTORY" + echo "system-packages missing at $SYSTEM_PACKAGES_DIR" sleep $ERROR_WAIT continue fi @@ -60,7 +61,7 @@ do elif [ -f buildmaster/do-everything ]; then echo "$(date) buildmaster everything requested, starting" echo "" - + rm buildmaster/do-everything build_prep @@ -81,7 +82,7 @@ do else echo "$(date) starting buildmaster update" echo "" - + build_prep buildmaster update fi @@ -117,17 +118,20 @@ do --system-packages-directory $SYSTEM_PACKAGES_DIR \ --check-package-repository-consistency \ --create-package-repository "$REPO_DIR" \ + --storage-backend-config "$STORAGE_BACKEND_CONFIG" \ $SIGFLAGS \ - > "$REPO_DIR/report.txt" 2>&1 + > "$OUTPUT_DIR/report.txt" 2>&1 - echo "" if [ $? -ne 0 ] then + echo "" echo "$(date) create repo failed, waiting $ERROR_WAIT" sleep $ERROR_WAIT rm -f /tmp/haiku-secret.key; continue fi + + echo "" rm -f /tmp/haiku-secret.key; if [ ! -z "$REPOSITORY_TRIGGER_URL" ] diff --git a/buildmaster/frontend/Dockerfile b/buildmaster/frontend/Dockerfile index 353d2531..e0a0220f 100644 --- a/buildmaster/frontend/Dockerfile +++ b/buildmaster/frontend/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/nginx:alpine -RUN mkdir /var/sources /var/instances /var/lib/buildmaster-frontend +RUN mkdir /var/buildmaster /var/lib/buildmaster-frontend -VOLUME ["/var/sources", "/var/instances"] +VOLUME ["/var/buildmaster"] COPY configs/buildmaster-frontend.conf /etc/nginx/conf.d/default.conf COPY www/. /var/lib/buildmaster-frontend/ diff --git a/buildmaster/frontend/configs/buildmaster-frontend.conf b/buildmaster/frontend/configs/buildmaster-frontend.conf index 10d83e0a..4cc55236 100644 --- a/buildmaster/frontend/configs/buildmaster-frontend.conf +++ b/buildmaster/frontend/configs/buildmaster-frontend.conf @@ -13,7 +13,7 @@ server { index buildmaster.html; location ~ ^/(.*?)/(.*?)/(.*)$ { - alias /var/instances/$1/$2/output/$3; + alias /var/buildmaster/$1/$2/output/$3; location ~ ^/(.*?)/(.*?)/(buildruns/(last_buildrun|buildruns.txt))$ { add_header Cache-Control "no-cache"; diff --git a/pyproject.toml b/pyproject.toml index 08c20d80..947512f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ python = ">=3.8,<4.0" black = "^23.7.0" isort = "^5.12.0" pylint = "^2.17.4" +boto3 = "^1.35.3" [tool.poetry.group.dev.dependencies] pytest = "^7.2.0" diff --git a/storage-backend-sample.json b/storage-backend-sample.json new file mode 100644 index 00000000..ea5fbc0c --- /dev/null +++ b/storage-backend-sample.json @@ -0,0 +1,8 @@ +{ + "backend_type": "s3", + "endpoint_url": "https://storage.example.org", + "access_key_id": "example_key", + "secret_access_key": "example_secret", + "bucket_name": "haikuports", + "prefix": "master/x86_64/current" +}