Skip to content

Commit

Permalink
feat: use finch-core to install core dependencies
Browse files Browse the repository at this point in the history
This change refactors the build system to use finch-core's dependency
mechanism for installing and verifying the base OS image, rootfs
archive, and Lima bundle needed for macOS and Windows platforms.

Signed-off-by: Austin Vazquez <[email protected]>
  • Loading branch information
austinvazquez committed Jun 26, 2024
1 parent 94b24d6 commit 364a557
Show file tree
Hide file tree
Showing 6 changed files with 90 additions and 158 deletions.
24 changes: 0 additions & 24 deletions .github/bin/update-os-image.sh

This file was deleted.

18 changes: 3 additions & 15 deletions .github/workflows/sync-submodules-and-deps.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,18 +34,6 @@ jobs:
run: |
git submodule update --remote
- name: Update OS image
run: |
./.github/bin/update-os-image.sh
- name: Update lima dependencies url
run: |
./deps/finch-core/bin/update-deps.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
- name: Update rootfs
run: |
./deps/finch-core/bin/update-rootfs.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
- name: Create PR
uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0
with:
Expand All @@ -55,9 +43,9 @@ jobs:
body: >
Bump for Finch's dependencies.
Currently, this updates values based on artifacts defined in the finch-core repository,
such as: operating system images, and the version of Lima (and it's dependencies),
which are bundled as `lima-and-qemu.macos-*.*.tar.gz` archives.
Updates finch-core submodule which defines the core dependencies for Finch on each respective
platform such as: operating system images and Lima bundle (Lima version and it's dependencies)
as `lima-and-qemu.macos-*.*.tar.gz` archives.
Since updating the Lima version may also update the default version of the `nerdctl-full`
archive (which contains nerdctl, containerd, buildkit, etc.), this may require a manual
Expand Down
138 changes: 20 additions & 118 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,24 @@ PREFIX ?= $(CURDIR)/_output
DEST := $(shell echo "$(DESTDIR)/$(PREFIX)" | sed 's:///*:/:g; s://*$$::')
BINDIR ?= /usr/local/bin
OUTDIR ?= $(CURDIR)/_output
OS_OUTDIR ?= $(OUTDIR)/os

OUTPUT_DIRECTORIES := $(OUTDIR) $(OS_OUTDIR)
$(OUTPUT_DIRECTORIES):
@mkdir -p $@

PACKAGE := github.com/runfinch/finch
BINARYNAME := finch
LIMA_FILENAME := lima
LIMA_EXTENSION := .tar.gz

LIMA_HOME := $(DEST)/lima/data
# Created by the CLI after installation, only used in uninstall step
LIMA_VDE_SUDOERS_FILE := /etc/sudoers.d/finch-lima
# Final installation prefix for vde created by CLI after installation, only used in uninstall step
VDE_INSTALL ?= /opt/finch
UNAME := $(shell uname -m)
ARCH ?= $(UNAME)
SUPPORTED_ARCH = false
CORE_VDE_PREFIX ?= $(OUTDIR)/dependencies/vde/opt/finch
LICENSEDIR := $(OUTDIR)/license-files
VERSION := $(shell git describe --match 'v[0-9]*' --dirty='.modified' --always --tags)
GITCOMMIT := $(shell git rev-parse HEAD)$(shell test -z "$(git status --porcelain)" || echo .m)
Expand All @@ -30,9 +34,6 @@ MIN_MACOS_VERSION ?= 11.0
GOOS ?= $(shell $(GO) env GOOS)
ifeq ($(GOOS),windows)
BINARYNAME := $(addsuffix .exe, $(BINARYNAME))
sha = sha256sum
else
sha = shasum -a 256
endif

.DEFAULT_GOAL := all
Expand All @@ -42,129 +43,31 @@ REGISTRY ?= ""
ifneq (,$(findstring arm64,$(ARCH)))
SUPPORTED_ARCH = true
LIMA_ARCH = aarch64
# From https://dl.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/aarch64/images/
FINCH_OS_BASENAME ?= Fedora-Cloud-Base-40-1.14.aarch64-20240514214641.qcow2
LIMA_URL ?= https://deps.runfinch.com/aarch64/lima-and-qemu.macos-aarch64.1715678889.tar.gz
else ifneq (,$(findstring x86_64,$(ARCH)))
SUPPORTED_ARCH = true
LIMA_ARCH = x86_64
# From https://dl.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/
FINCH_OS_BASENAME ?= Fedora-Cloud-Base-40-1.14.x86_64-20240514214655.qcow2
LIMA_URL ?= https://deps.runfinch.com/x86-64/lima-and-qemu.macos-x86_64.1715678889.tar.gz
FINCH_ROOTFS_URL ?= https://deps.runfinch.com/common/x86-64/finch-rootfs-production-amd64-1715724303.tar.gz
FINCH_ROOTFS_BASENAME := $(notdir $(FINCH_ROOTFS_URL))
endif

FINCH_OS_HASH := `$(sha) $(OUTDIR)/os/$(FINCH_OS_BASENAME) | cut -d ' ' -f 1`
FINCH_OS_DIGEST := "sha256:$(FINCH_OS_HASH)"
FINCH_OS_IMAGE_LOCATION_ROOT ?= $(DEST)
FINCH_OS_IMAGE_LOCATION ?= $(FINCH_OS_IMAGE_LOCATION_ROOT)/os/$(FINCH_OS_BASENAME)

# TODO: Windows PoC extracting rootfs...
FINCH_ROOTFS_HASH := `$(sha) $(OUTDIR)/os/$(FINCH_ROOTFS_BASENAME) | cut -d ' ' -f 1`
FINCH_ROOTFS_DIGEST := "sha256:$(FINCH_ROOTFS_HASH)"
FINCH_ROOTFS_LOCATION_ROOT ?= $(DEST)/
FINCH_ROOTFS_LOCATION ?= $(FINCH_ROOTFS_LOCATION_ROOT)os/$(FINCH_ROOTFS_BASENAME)

.PHONY: arch-test
arch-test:
@if [ $(SUPPORTED_ARCH) != "true" ]; then echo "Unsupported architecture: $(ARCH)"; exit "1"; fi

.PHONY: all
ifeq ($(GOOS),windows)
all: arch-test finch finch-core-local finch.windows.yaml networks.yaml config.yaml
BUILD_OS ?= $(OS)
FINCH_CORE_DIR := $(CURDIR)/deps/finch-core
ifeq ($(BUILD_OS), Windows_NT)
include Makefile.windows
else
all: arch-test finch finch-core finch.yaml networks.yaml config.yaml lima-and-qemu
include Makefile.darwin
endif

.PHONY: all-local
all-local: arch-test networks.yaml config.yaml lima-and-qemu local-core finch.yaml

.PHONY: finch-core
finch-core:
cd deps/finch-core && \
FINCH_OS_AARCH64_URL="$(FINCH_OS_AARCH64_URL)" \
VDE_TEMP_PREFIX=$(CORE_VDE_PREFIX) \
"$(MAKE)"

mkdir -p _output
cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)
rm -rf $(OUTDIR)/lima-template

.PHONY: finch-core-local
finch-core-local:
cd deps/finch-core && \
FINCH_OS_x86_URL="$(FINCH_OS_x86_URL)" \
FINCH_OS_AARCH64_URL="$(FINCH_OS_AARCH64_URL)" \
VDE_TEMP_PREFIX=$(CORE_VDE_PREFIX) \
"$(MAKE)" all lima

mkdir -p _output
cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)
rm -rf $(OUTDIR)/lima-template

.PHONY: local-core
local-core:
cd deps/finch-core && \
FINCH_OS_x86_URL="$(FINCH_OS_x86_URL)" \
FINCH_OS_AARCH64_URL="$(FINCH_OS_AARCH64_URL)" \
VDE_TEMP_PREFIX=$(CORE_VDE_PREFIX) \
"$(MAKE)" lima lima-socket-vmnet

mkdir -p _output
cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)
cd deps/finch-core/src/lima/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)/lima
cd deps/finch-core/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)
cd deps/finch-core/src/lima/_output && tar -cf - * | tar -xvf - -C $(OUTDIR)/lima
rm -rf $(OUTDIR)/lima-template

.PHONY: lima-and-qemu
lima-and-qemu: networks.yaml
mkdir -p $(OUTDIR)/downloads
# download artifacts
curl -L $(LIMA_URL) > $(OUTDIR)/downloads/lima-and-qemu.tar.gz

# Untar LIMA
tar -xvf $(OUTDIR)/downloads/lima-and-qemu.tar.gz -C $(OUTDIR)/lima/

# Delete downloads
rm -rf $(OUTDIR)/downloads


FINCH_IMAGE_LOCATION ?=
FINCH_IMAGE_DIGEST ?=
ifeq ($(GOOS),windows)
# Because the path in windows /C:/<some-path> is not an Absolute path, prefix with file:/ which is handled by lima https://github.com/lima-vm/lima/blob/da1260dc87fb30345c3ee7bfb131c29646e26d10/pkg/downloader/downloader.go#L266
FINCH_IMAGE_LOCATION := "file:/$(FINCH_ROOTFS_LOCATION)"
FINCH_IMAGE_DIGEST := $(FINCH_ROOTFS_DIGEST)
else
FINCH_IMAGE_LOCATION := $(FINCH_OS_IMAGE_LOCATION)
FINCH_IMAGE_DIGEST := $(FINCH_OS_DIGEST)
endif
.PHONY: finch.yaml
finch.yaml: finch-core
mkdir -p $(OUTDIR)/os
# merge the appropriate YAMLs
cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' mac.yaml common.yaml > ../finch.yaml
cp finch.yaml $(OUTDIR)/os
# using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed
# this sed command uses the alternative separator of "|" because the image location uses "/"
sed -i.bak -e "s|<finch_image_location>|$(FINCH_IMAGE_LOCATION)|g" $(OUTDIR)/os/finch.yaml
sed -i.bak -e "s/<finch_image_arch>/$(LIMA_ARCH)/g" $(OUTDIR)/os/finch.yaml
sed -i.bak -e "s/<finch_image_digest>/$(FINCH_IMAGE_DIGEST)/g" $(OUTDIR)/os/finch.yaml

# TODO: Windows PoC - clean this up / consolidate
all: arch-test finch install.finch-core-dependencies finch.yaml networks.yaml config.yaml

.PHONY: install.finch-core-dependencies
install.finch-core-dependencies:
OUTDIR=$(OUTDIR) "$(MAKE)" -C $(FINCH_CORE_DIR) install.dependencies

.PHONY: finch.yaml
finch.windows.yaml: finch-core-local
mkdir -p $(OUTDIR)/os
# merge the appropriate YAMLs
cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' windows.yaml common.yaml > ../finch.windows.yaml
cp finch.windows.yaml $(OUTDIR)/os/finch.yaml
# using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed
# this sed command uses the alternative separator of "|" because the image location uses "/"
sed -i.bak -e "s|<finch_image_location>|$(FINCH_IMAGE_LOCATION)|g" $(OUTDIR)/os/finch.yaml
sed -i.bak -e "s/<finch_image_arch>/$(LIMA_ARCH)/g" $(OUTDIR)/os/finch.yaml
sed -i.bak -e "s/<finch_image_digest>/$(FINCH_IMAGE_DIGEST)/g" $(OUTDIR)/os/finch.yaml
finch.yaml: $(OS_OUTDIR)/finch.yaml

.PHONY: networks.yaml
networks.yaml:
Expand Down Expand Up @@ -402,7 +305,7 @@ mdlint-ctr:
ifeq ($(GOOS),windows)
clean:
-@rm -rf $(OUTDIR) 2>/dev/null || true
-@rm -rf ./deps/finch-core/_output || true
-@"$(MAKE)" -C $(FINCH_CORE_DIR) clean
-@rm ./*.tar.gz 2>/dev/null || true
-@rm ./*.qcow2 2>/dev/null || true
-@rm ./test-coverage.* 2>/dev/null || true
Expand All @@ -415,8 +318,7 @@ clean:
-sudo rm -rf "/private/var/run/finch-lima"
-sudo rm -rf "/private/etc/sudoers.d/finch-lima"
-@rm -rf $(OUTDIR) 2>/dev/null || true
-@rm -rf ./deps/finch-core/_output || true
-@rm -rf ./deps/finch-core/downloads/os/$(FINCH_OS_BASENAME) || true
-@$(MAKE) -C $(FINCH_CORE_DIR) clean
-@rm ./*.tar.gz 2>/dev/null || true
-@rm ./*.qcow2 2>/dev/null || true
-@rm ./test-coverage.* 2>/dev/null || true
Expand Down
34 changes: 34 additions & 0 deletions Makefile.darwin
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

# For Finch on macOS, the base OS location and digest are set
# based on the values set in deps/finch-core/deps/full-os.conf
-include $(FINCH_CORE_DIR)/deps/full-os.conf
ifeq ($(ARCH),x86_64)
FINCH_OS_BASENAME=$(X86_64_ARTIFACT)
FINCH_OS_DIGEST=$(X86_64_512_DIGEST)
else
FINCH_OS_BASENAME=$(AARCH64_ARTIFACT)
FINCH_OS_DIGEST=$(AARCH64_512_DIGEST)
endif

FINCH_IMAGE_LOCATION:=$(OS_OUTDIR)/$(FINCH_OS_BASENAME)
FINCH_IMAGE_DIGEST:="sha512:$(FINCH_OS_DIGEST)"

.PHONY: finch.yaml
finch.yaml: $(OS_OUTDIR)/finch.yaml

$(OS_OUTDIR)/finch.yaml: $(OS_OUTDIR) finch.yaml.d/common.yaml finch.yaml.d/mac.yaml
# merge the appropriate YAMLs into a temporary finch.yaml file on the current working directory
cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' mac.yaml common.yaml > ../finch.yaml.temp

# using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed
# this sed command uses the alternative separator of "|" because the image location uses "/"
sed -i.bak -e "s|<finch_image_location>|$(FINCH_IMAGE_LOCATION)|g" finch.yaml.temp
sed -i.bak -e "s/<finch_image_arch>/$(LIMA_ARCH)/g" finch.yaml.temp
sed -i.bak -e "s/<finch_image_digest>/$(FINCH_IMAGE_DIGEST)/g" finch.yaml.temp

# Replacement was successful, so cleanup .bak
@rm finch.yaml.temp.bak

mv finch.yaml.temp $@
32 changes: 32 additions & 0 deletions Makefile.windows
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

# For Finch on Windows, the rootfs location and digest are set
# based on the values in deps/finch-core/deps/rootfs.conf
-include $(FINCH_CORE_DIR)/deps/rootfs.conf
ifeq ($(ARCH),x86_64)
FINCH_ROOTFS_BASENAME ?= $(X86_64_ARTIFACT)
FINCH_ROOTFS_DIGEST ?= $(X86_64_512_DIGEST)
else
$(error Finch on Windows ARM not supported)
endif

FINCH_IMAGE_LOCATION := $(OS_OUTDIR)/$(FINCH_ROOTFS_BASENAME)
FINCH_IMAGE_DIGEST := "sha512:$(FINCH_ROOTFS_DIGEST)"

$(OS_OUTDIR)/finch.yaml: $(OS_OUTDIR) finch.yaml.d/common.yaml finch.yaml.d/windows.yaml
# merge the appropriate YAMLs into a temporary finch.yaml file on the current working directory
cd finch.yaml.d && yq eval-all '. as $$item ireduce ({}; . *+ $$item)' windows.yaml common.yaml > ../finch.yaml.temp

# using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed
# this sed command uses the alternative separator of "|" because the image location uses "/"
# Because the path in windows /C:/<some-path> is not an Absolute path, prefix with file:/ which is handled by lima
# https://github.com/lima-vm/lima/blob/da1260dc87fb30345c3ee7bfb131c29646e26d10/pkg/downloader/downloader.go#L266
sed -i.bak -e "s|<finch_image_location>|file://$(FINCH_IMAGE_LOCATION)|g" finch.yaml.temp
sed -i.bak -e "s/<finch_image_arch>/$(LIMA_ARCH)/g" finch.yaml.temp
sed -i.bak -e "s/<finch_image_digest>/$(FINCH_IMAGE_DIGEST)/g" finch.yaml.temp

# Replacement was successful, so cleanup .bak
@rm finch.yaml.temp.bak

mv finch.yaml.temp $@

0 comments on commit 364a557

Please sign in to comment.