Merge pull request #291 from mmlr/s3storage

Implement S3 Object Storage for Package Repositories

HaikuPorter/BuildMaster.py

@@ -43,18 +43,18 @@ def filter(self, record):
 
 
 class ScheduledBuild(object):
-	def __init__(self, port, portsTreePath, requiredPackageIDs, packagesPath,
-		presentDependencyPackages):
+	def __init__(self, port, portsTreePath, missingPackageIDs,
+		packageRepository, presentDependencyPackages):
 		self.port = port
 		self.recipeFilePath \
 			= os.path.relpath(port.recipeFilePath, portsTreePath)
 		self.resultingPackages \
 			= [package.hpkgName for package in self.port.packages]
-		self.packagesPath = packagesPath
+		self.packageRepository = packageRepository
 		self.requiredPackages = presentDependencyPackages
 		self.requiredPackageIDs = [
 			os.path.basename(path) for path in presentDependencyPackages]
-		self.missingPackageIDs = set(requiredPackageIDs)
+		self.missingPackageIDs = set(missingPackageIDs)
 		self.buildNumbers = []
 		self.lost = False
 
@@ -69,7 +69,7 @@ def packageCompleted(self, package, available):
 				self.missingPackageIDs.remove(packageID)
 				self.requiredPackageIDs.append(package.hpkgName)
 				self.requiredPackages.append(
-					os.path.join(self.packagesPath, package.hpkgName))
+					self.packageRepository.packagePath(package.hpkgName))
 			else:
 				self.lost = True
 
@@ -151,15 +151,15 @@ def status(self):
 
 
 class BuildMaster(object):
-	def __init__(self, portsTreePath, packagesPath, options):
+	def __init__(self, portsTreePath, packageRepository, options):
 		self.portsTreePath = portsTreePath
 		self._fillPortsTreeInfo()
 
 		self.activeBuilders = []
 		self.reconnectingBuilders = []
 		self.lostBuilders = []
 		self.availableBuilders = []
-		self.packagesPath = packagesPath
+		self.packageRepository = packageRepository
 		self.masterBaseDir = os.path.realpath('buildmaster')
 		self.builderBaseDir = os.path.join(self.masterBaseDir, 'builders')
 		self.buildOutputBaseDir = getOption('buildMasterOutputDir')
@@ -230,9 +230,9 @@ def __init__(self, portsTreePath, packagesPath, options):
 
 				builder = None
 				try:
-					builder = RemoteBuilderSSH(configFilePath, packagesPath,
-						self.buildOutputBaseDir, self.portsTreeOriginURL,
-						self.portsTreeHead)
+					builder = RemoteBuilderSSH(configFilePath,
+						packageRepository, self.buildOutputBaseDir,
+						self.portsTreeOriginURL, self.portsTreeHead)
 				except Exception as exception:
 					self.logger.error('failed to add builder from config '
 						+ configFilePath + ':' + str(exception))
@@ -247,7 +247,7 @@ def __init__(self, portsTreePath, packagesPath, options):
 			for i in range(0, self.localBuilders):
 				builder = None
 				try:
-					builder = LocalBuilder(str(i), packagesPath,
+					builder = LocalBuilder(str(i), packageRepository,
 						self.buildOutputBaseDir, options)
 				except Exception as exception:
 					self.logger.error('failed to add local builder: '
@@ -292,20 +292,20 @@ def addSkipped(self, port, reason):
 		self.skippedBuilds.append(skippedBuild)
 		self._reportStatus()
 
-	def schedule(self, port, requiredPackageIDs, presentDependencyPackages):
+	def schedule(self, port, missingPackageIDs, presentDependencyPackages):
 		# Skip builds that would overwrite existing packages.
 		for package in port.packages:
-			packagePath = os.path.join(self.packagesPath, package.hpkgName)
-			if not os.path.exists(packagePath):
+			if not self.packageRepository.hasPackage(package.hpkgName):
 				continue
 
-			self.addSkipped(port, 'some packages already exist at '
-				+ self.packagesPath + ', revision bump required')
+			self.addSkipped(port, 'some packages already exist in package'
+				+ ' repository, revision bump required')
 			return
 
 		self.logger.info('scheduling build of ' + port.versionedName)
 		scheduledBuild = ScheduledBuild(port, self.portsTreePath,
-			requiredPackageIDs, self.packagesPath, presentDependencyPackages)
+			missingPackageIDs, self.packageRepository,
+			presentDependencyPackages)
 
 		if scheduledBuild.buildable:
 			self.scheduledBuilds.append(scheduledBuild)

HaikuPorter/Builders/LocalBuilder.py

@@ -14,12 +14,12 @@
 
 
 class LocalBuilder(object):
-	def __init__(self, name, packagesPath, outputBaseDir, options):
+	def __init__(self, name, packageRepository, outputBaseDir, options):
 		self.options = options
 		self.name = name
 		self.buildCount = 0
 		self.failedBuilds = 0
-		self.packagesPath = packagesPath
+		self.packageRepository = packageRepository
 		self.state = BuilderState.AVAILABLE
 		self.currentBuild = None
 

HaikuPorter/Builders/RemoteBuilderSSH.py

@@ -15,6 +15,7 @@
 # These usages kinda need refactored
 from ..ConfigParser import ConfigParser
 from ..Configuration import Configuration
+from ..Options import getOption
 from .Builder import BuilderState
 
 try:
@@ -23,14 +24,14 @@
 	paramiko = None
 
 class RemoteBuilderSSH(object):
-	def __init__(self, configFilePath, packagesPath, outputBaseDir,
+	def __init__(self, configFilePath, packageRepository, outputBaseDir,
 			portsTreeOriginURL, portsTreeHead):
 		self._loadConfig(configFilePath)
 		self.availablePackages = []
 		self.visiblePackages = []
 		self.portsTreeOriginURL = portsTreeOriginURL
 		self.portsTreeHead = portsTreeHead
-		self.packagesPath = packagesPath
+		self.packageRepository = packageRepository
 
 		if not paramiko:
 			raise Exception('paramiko unavailable')
@@ -231,13 +232,20 @@ def _getAvailablePackages(self):
 
 	def _removeObsoletePackages(self):
 		cachePath = self.config['portstree']['packagesCachePath']
+		systemPackagesDirectory = getOption('systemPackagesDirectory')
+
 		for entry in list(self.availablePackages):
-			if not os.path.exists(os.path.join(self.packagesPath, entry)):
-				self.logger.info(
-					'removing obsolete package {} from cache'.format(entry))
-				entryPath = cachePath + '/' + entry
-				self.sftpClient.remove(entryPath)
-				self.availablePackages.remove(entry)
+			if self.packageRepository.hasPackage(entry):
+				continue
+
+			if os.path.exists(os.path.join(systemPackagesDirectory, entry)):
+				continue
+
+			self.logger.info(
+				'removing obsolete package {} from cache'.format(entry))
+			entryPath = cachePath + '/' + entry
+			self.sftpClient.remove(entryPath)
+			self.availablePackages.remove(entry)
 
 	def _setupForBuilding(self):
 		if self.state == BuilderState.AVAILABLE:
@@ -331,11 +339,11 @@ def runBuild(self):
 				self.buildLogger.info('download package ' + package.hpkgName
 					+ ' from builder')
 
-				packageFile = os.path.join(self.packagesPath, package.hpkgName)
-				downloadFile = packageFile + '.download'
-				self._getFile(self.config['portstree']['packagesPath'] + '/'
-						+ package.hpkgName, downloadFile)
-				os.rename(downloadFile, packageFile)
+				remotePath = self.config['portstree']['packagesPath'] + '/' \
+					+ package.hpkgName
+				with self.sftpClient.file(remotePath, 'r') as remoteFile:
+					self.packageRepository.writePackage(package.hpkgName,
+						remoteFile)
 
 			self._purgePort(scheduledBuild)
 			self._clearVisiblePackages()
@@ -374,12 +382,7 @@ def _symlink(self, sourcePath, destPath):
 		self.sftpClient.symlink(sourcePath, destPath)
 
 	def _move(self, sourcePath, destPath):
-		# Unfortunately we can't use SFTPClient.rename as that uses the rename
-		# command (vs. posix-rename) which uses hardlinks which fail on BFS
-		(output, channel) = self._remoteCommand('mv "' + sourcePath + '" "'
-			+ destPath + '"')
-		if channel.recv_exit_status() != 0:
-			raise IOError('failed moving {} to {}'.format(sourcePath, destPath))
+		self.sftpClient.posix_rename(sourcePath, destPath)
 
 	def _openRemoteFile(self, path, mode):
 		return self.sftpClient.open(path, mode)
@@ -430,7 +433,9 @@ def _makePackageAvailable(self, packagePath):
 			= self.config['portstree']['packagesCachePath'] + '/' + packageName
 		uploadPath = entryPath + '.upload'
 
-		self._putFile(packagePath, uploadPath)
+		with self.sftpClient.file(uploadPath, 'w') as remoteFile:
+			self.packageRepository.readPackage(packagePath, remoteFile)
+
 		self._move(uploadPath, entryPath)
 
 		self.availablePackages.append(packageName)

HaikuPorter/Main.py

@@ -109,7 +109,11 @@ def run(self, args):
 
 		if self.options.buildMaster:
 			from .BuildMaster import BuildMaster
-			self.buildMaster = BuildMaster(self.treePath, self.packagesPath,
+
+			packageRepository = PackageRepository(self.packagesPath,
+				None, self.options.quiet, self.options.verbose)
+
+			self.buildMaster = BuildMaster(self.treePath, packageRepository,
 				self.options)
 
 			self.options.allDependencies = True

HaikuPorter/Options.py

@@ -305,6 +305,9 @@ def parseOptions():
 	advanced_flags.add_option('--sign-package-repository-privkey-pass', action='store',
 		type='string', dest='packageRepositorySignPrivateKeyPass', default=None,
 		help='sign the package repository with the given minisign password')
+	advanced_flags.add_option('--storage-backend-config', action='store',
+		dest='storageBackendConfig', default=None, type='string',
+		help='use the given file as the storage backend config'),
 	advanced_flags.add_option('--check-package-repository-consistency',
 		action='store_true', dest='checkPackageRepositoryConsistency',
 		default=False, help='check consistency of package repository by'

HaikuPorter/Package.py

@@ -180,18 +180,6 @@ def removeDependencyInfoFromRepository(self, repositoryPath):
 		if os.path.exists(dependencyInfoFile):
 			os.remove(dependencyInfoFile)
 
-	def obsoletePackage(self, packagesPath):
-		"""Moves the package-file into the 'obsolete' sub-directory"""
-
-		obsoleteDir = packagesPath + '/.obsolete'
-		packageFile = packagesPath + '/' + self.hpkgName
-		if os.path.exists(packageFile):
-			print('\tobsoleting package ' + self.hpkgName)
-			obsoletePackage = obsoleteDir + '/' + self.hpkgName
-			if not os.path.exists(obsoleteDir):
-				os.mkdir(obsoleteDir)
-			os.rename(packageFile, obsoletePackage)
-
 	def generateDependencyInfoWithoutProvides(self, dependencyInfoPath,
 											  requiresToUse):
 		"""Create a .DependencyInfo file that doesn't include any provides

HaikuPorter/PackageInfo.py

@@ -96,7 +96,8 @@ def _initializeCache(cls):
 					entry = pickle.load(cacheFile)
 					path = entry['path']
 					if not os.path.exists(path) \
-						or os.path.getmtime(path) > entry['modifiedTime']:
+						or (os.path.getsize(path) != 0 \
+							and os.path.getmtime(path) > entry['modifiedTime']):
 						prune = True
 						continue