From 7852e83882104a3523770f7f1c0dc9bad0a2fd05 Mon Sep 17 00:00:00 2001
From: Ethan Li <lietk12@gmail.com>
Date: Thu, 16 May 2024 01:51:51 -0400
Subject: [PATCH] Fix export of hardlinks from archives, and auto-download
 files needed for staging (#210)

* Fix export of hardlinks from archives & OCI images

* Fix handling of missing download files when staging a pallet

* Bump version in `CHANGELOG.md`
---
 CHANGELOG.md                                 |  7 ++
 internal/app/forklift/bundles.go             | 71 ++++++++++++--------
 internal/app/forklift/cli/pallets-staging.go |  6 ++
 3 files changed, 57 insertions(+), 27 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 64bcd6f4..38bb2dcf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.7.2-alpha.6 - 2024-05-16
+
+### Fixed
+
+- (cli) Hard links should now be handled correctly when they need to be exported from downloaded archives or OCI images.
+- (cli) Staging a pallet now includes the download of any missing files, OCI container images, and repos required for staging.
+
 ## 0.7.2-alpha.5 - 2024-05-15
 
 ### Fixed
diff --git a/internal/app/forklift/bundles.go b/internal/app/forklift/bundles.go
index 109ba01e..c5e655f1 100644
--- a/internal/app/forklift/bundles.go
+++ b/internal/app/forklift/bundles.go
@@ -325,7 +325,7 @@ func exportArchiveFile(
 			"unrecognized archive file type: %s (.%s)", kind.MIME.Value, kind.Extension,
 		)
 	}
-	if err = extractFile(archiveReader, export.Source, exportPath); err != nil {
+	if err = extractFromArchive(archiveReader, export.Source, exportPath); err != nil {
 		return errors.Wrapf(
 			err, "couldn't extract %s from cached download archive %s to %s",
 			export.Source, export.URL, exportPath,
@@ -365,7 +365,7 @@ func determineFileType(
 	return filetype.MatchReader(archiveFile)
 }
 
-func extractFile(tarReader *tar.Reader, sourcePath, exportPath string) error {
+func extractFromArchive(tarReader *tar.Reader, sourcePath, exportPath string) error {
 	if sourcePath == "/" || sourcePath == "." {
 		sourcePath = ""
 	}
@@ -381,32 +381,49 @@ func extractFile(tarReader *tar.Reader, sourcePath, exportPath string) error {
 			!strings.HasPrefix(header.Name, sourcePath+"/") {
 			continue
 		}
-		targetPath := path.Join(exportPath, strings.TrimPrefix(header.Name, sourcePath))
-		switch header.Typeflag {
-		default:
-			return errors.Errorf(
-				"unknown type of file %s in archive: %b", header.Name, header.Typeflag,
+
+		if err = extractFile(header, tarReader, sourcePath, exportPath); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func extractFile(
+	header *tar.Header, tarReader *tar.Reader, sourcePath, exportPath string,
+) error {
+	targetPath := path.Join(exportPath, strings.TrimPrefix(header.Name, sourcePath))
+	switch header.Typeflag {
+	default:
+		return errors.Errorf("unknown type of file %s in archive: %b", header.Name, header.Typeflag)
+	case tar.TypeDir:
+		if err := EnsureExists(filepath.FromSlash(targetPath)); err != nil {
+			return errors.Wrapf(
+				err, "couldn't export directory %s from archive to %s", header.Name, targetPath,
+			)
+		}
+	case tar.TypeReg:
+		if err := extractRegularFile(header, tarReader, sourcePath, targetPath); err != nil {
+			return errors.Wrapf(
+				err, "couldn't export regular file %s from archive to %s", header.Name, targetPath,
+			)
+		}
+	case tar.TypeSymlink:
+		if err := os.Symlink(
+			filepath.FromSlash(header.Linkname), filepath.FromSlash(targetPath),
+		); err != nil {
+			return errors.Wrapf(
+				err, "couldn't export symlink %s from archive to %s", header.Name, targetPath,
+			)
+		}
+	case tar.TypeLink:
+		if err := os.Link(
+			filepath.FromSlash(path.Join(exportPath, strings.TrimPrefix(header.Linkname, sourcePath))),
+			filepath.FromSlash(targetPath),
+		); err != nil {
+			return errors.Wrapf(
+				err, "couldn't export hardlink %s from archive to %s", header.Name, targetPath,
 			)
-		case tar.TypeDir:
-			if err = EnsureExists(filepath.FromSlash(targetPath)); err != nil {
-				return errors.Wrapf(
-					err, "couldn't export directory %s from archive to %s", header.Name, targetPath,
-				)
-			}
-		case tar.TypeReg:
-			if err = extractRegularFile(header, tarReader, sourcePath, targetPath); err != nil {
-				return errors.Wrapf(
-					err, "couldn't export regular file %s from archive to %s", header.Name, targetPath,
-				)
-			}
-		case tar.TypeSymlink:
-			if err = os.Symlink(
-				filepath.FromSlash(header.Linkname), filepath.FromSlash(targetPath),
-			); err != nil {
-				return errors.Wrapf(
-					err, "couldn't export symlink %s from archive to %s", header.Name, targetPath,
-				)
-			}
 		}
 	}
 	return nil
diff --git a/internal/app/forklift/cli/pallets-staging.go b/internal/app/forklift/cli/pallets-staging.go
index 2c2f86c9..682205e2 100644
--- a/internal/app/forklift/cli/pallets-staging.go
+++ b/internal/app/forklift/cli/pallets-staging.go
@@ -17,6 +17,12 @@ func StagePallet(
 	exportPath, toolVersion, bundleMinVersion, newBundleForkliftVersion string,
 	skipImageCaching, parallel, ignoreToolVersion bool,
 ) (index int, err error) {
+	if err = CacheStagingRequirements(
+		pallet, repoCache.Path(), repoCache, dlCache, false, parallel,
+	); err != nil {
+		return 0, errors.Wrap(err, "couldn't cache requirements for staging the pallet")
+	}
+
 	index, err = stageStore.AllocateNew()
 	if err != nil {
 		return 0, errors.Wrap(err, "couldn't allocate a directory for staging")