Skip to content

Commit

Permalink
make registry configurable & refactor layers fetching to be better te…
Browse files Browse the repository at this point in the history
…sted
  • Loading branch information
d8vjork committed Nov 10, 2023
1 parent 28e11e0 commit 964dae7
Show file tree
Hide file tree
Showing 15 changed files with 6,289 additions and 89 deletions.
19 changes: 17 additions & 2 deletions config/sidecar-local.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,27 @@

return [

/**
* Images registry from where to pull all base images needed for Lambda functions.
*
* Make sure this registry is public otherwise ensure you have the right permissions.
*/
'registry' => 'public.ecr.aws/lambda',

/**
* The port used to call any Dockerised service function.
*
*
* Make sure this port isn't used by any other service within your computer.
*/

'port' => 6000,

/**
* Base path from where Lambda functions handlers/code are stored.
*
* From project's root path. For e.g. resources/sidecar.
*
* Null will set them to be parent directory from sidecar function PHP (setBasePath).
*/
'path' => 'resources/sidecar',

];
76 changes: 16 additions & 60 deletions src/Commands/DeployLocal.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@

namespace OpenSoutheners\SidecarLocal\Commands;

use Hammerstone\Sidecar\Clients\LambdaClient;
use Hammerstone\Sidecar\LambdaFunction;
use Hammerstone\Sidecar\Sidecar;
use Illuminate\Console\Command;
use Illuminate\Filesystem\Filesystem;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Str;
use OpenSoutheners\SidecarLocal\SidecarLayers;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Yaml\Yaml;
use ZipArchive;

class DeployLocal extends Command
{
Expand Down Expand Up @@ -88,7 +88,7 @@ protected function buildYamlServices(string $basePath): array
$functionLayersPath = null;

if (count($functionLayers) > 0) {
$functionLayersPath = $this->grabFunctionRequiredLayers($basePath, $functionClassName, $functionLayers);
$functionLayersPath = $this->grabFunctionRequiredLayers($lambdaFunction, $functionLayers);
}

$functionAwsCallName = $lambdaFunction->nameWithPrefix();
Expand All @@ -97,7 +97,8 @@ protected function buildYamlServices(string $basePath): array
$image = Str::replaceLast('.x', '', $lambdaFunction->runtime());
$imageRuntime = preg_replace('/\d|/', '', $image);
$imageRuntimeTag = filter_var($image, FILTER_SANITIZE_NUMBER_INT);
$awsLambdaImage = "amazon/aws-lambda-{$imageRuntime}:{$imageRuntimeTag}";
$awsLambdaImage = config('sidecar-local.registry', 'public.ecr.aws/lambda');
$awsLambdaImage .= "/{$imageRuntime}:{$imageRuntimeTag}-{$lambdaFunction->architecture()}";

$services[$serviceName] = [
'container_name' => $functionClassName,
Expand All @@ -111,7 +112,7 @@ protected function buildYamlServices(string $basePath): array
'command' => $lambdaFunction->handler(),
'volumes' => ["./{$functionClassName}:/var/task"],
'labels' => [
"traefik.enable=true",
'traefik.enable=true',
"traefik.http.routers.{$serviceName}.entrypoints=web",
"traefik.http.routers.{$serviceName}.rule=Host(`localhost`) && Path(`/2015-03-31/functions/{$functionAwsCallName}:active/invocations`)",
"traefik.http.middlewares.{$serviceName}-replacepath.replacepath.path=/2015-03-31/functions/function/invocations",
Expand All @@ -131,56 +132,11 @@ protected function buildYamlServices(string $basePath): array
/**
* Get function required Lambda layers from AWS cloud.
*/
protected function grabFunctionRequiredLayers(string $basePath, string $functionClass, array $arns): string
protected function grabFunctionRequiredLayers(LambdaFunction $function, array $arns): string
{
// We need to set environment any value rather than "local"
// to be able to grab layers from AWS cloud.
config(['sidecar.env' => 'deploying']);

$zipArchive = new ZipArchive();

$functionLayerDirectory = "{$basePath}/layers/{$functionClass}";
$this->filesystem->ensureDirectoryExists($functionLayerDirectory);

$progressBar = $this->getOutput()->createProgressBar(count($arns));
$progressBar->start();

foreach ($arns as $arn) {
$layerDirectoryName = Str::of($arn)->afterLast(':layer:')->replaceLast(':', '/');
$layerZipFileName = $layerDirectoryName->afterLast('/')->append('.zip')->value();
$layerDirectoryName = $layerDirectoryName->value();

$layerTmpDirectoryPath = storage_path("tmp/layers/{$layerDirectoryName}");
$this->filesystem->ensureDirectoryExists($layerTmpDirectoryPath);
$layerTmpFilePath = "{$layerTmpDirectoryPath}/{$layerZipFileName}";

if (! $this->filesystem->exists($layerTmpFilePath)) {
$layerData = app(LambdaClient::class)->getLayerVersionByArn(['Arn' => $arn])->toArray();

$layerLocation = data_get($layerData, 'Content.Location');

if (! $layerLocation || ! $this->filesystem->copy($layerLocation, $layerTmpFilePath)) {
$this->warn("Cannot download layer for function '{$functionClass}'.");

continue;
}
}

$zipArchive->open($layerTmpFilePath);
$zipArchive->extractTo($functionLayerDirectory);
$zipArchive->close();

$progressBar->advance();
}

$progressBar->clear();

config(['sidecar.env' => 'local']);

return Str::of($functionLayerDirectory)
->replaceFirst($basePath, '.')
->append(':/opt:ro')
->value();
return app(SidecarLayers::class)->fetch($function, $arns, $progressBar);
}

/**
Expand All @@ -200,17 +156,17 @@ protected function writeComposeFile(string $basePath, array $services): bool
],
],
'ports' => [
config('sidecar-local.port', 6000).":80",
"8080:8080",
config('sidecar-local.port', 6000).':80',
'8080:8080',
],
'command' => [
"--api.insecure=true",
"--providers.docker=true",
"--providers.docker.exposedbydefault=false",
"--entryPoints.web.address=:80",
'--api.insecure=true',
'--providers.docker=true',
'--providers.docker.exposedbydefault=false',
'--entryPoints.web.address=:80',
],
'volumes' => [
"/var/run/docker.sock:/var/run/docker.sock",
'/var/run/docker.sock:/var/run/docker.sock',
],
],
], $services),
Expand Down Expand Up @@ -262,7 +218,7 @@ protected function getOptions()
{
return [
['run', null, InputOption::VALUE_NONE, 'Run created services in Docker'],
['path', 'o', InputOption::VALUE_OPTIONAL, 'Relative path to the root where to write resulted docker-compose.yml file', 'resources/sidecar'],
['path', 'o', InputOption::VALUE_OPTIONAL, 'Relative path to the root where to write resulted docker-compose.yml file', config('sidecar-local.path', 'resources/sidecar')],
['stop', null, InputOption::VALUE_NONE, 'Stop all Docker services previously created using this command on the path'],
];
}
Expand Down
8 changes: 8 additions & 0 deletions src/Exceptions/SidecarIncompatibleFunctionArchitecture.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<?php

namespace OpenSoutheners\SidecarLocal\Exceptions;

class SidecarIncompatibleFunctionArchitecture extends \Exception
{
public const MESSAGE = 'Architecture "%s" not supported by function "%s". Supported "%s".';
}
8 changes: 8 additions & 0 deletions src/Exceptions/SidecarIncompatibleFunctionRuntime.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<?php

namespace OpenSoutheners\SidecarLocal\Exceptions;

class SidecarIncompatibleFunctionRuntime extends \Exception
{
public const MESSAGE = 'Runtime "%s" not supported by function "%s". Supported "%s".';
}
138 changes: 138 additions & 0 deletions src/SidecarLayers.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
<?php

namespace OpenSoutheners\SidecarLocal;

use Hammerstone\Sidecar\Clients\LambdaClient;
use Hammerstone\Sidecar\LambdaFunction;
use Illuminate\Filesystem\Filesystem;
use Illuminate\Support\Str;
use OpenSoutheners\SidecarLocal\Exceptions\SidecarIncompatibleFunctionArchitecture;
use OpenSoutheners\SidecarLocal\Exceptions\SidecarIncompatibleFunctionRuntime;
use Symfony\Component\Console\Helper\ProgressBar;
use ZipArchive;

class SidecarLayers
{
public function __construct(
protected LambdaClient $client,
protected Filesystem $filesystem
) {
//
}

/**
* Fetch function's required layers then return volume mapping otherwise false.
*/
public function fetch(LambdaFunction $function, array $arns, ProgressBar $progressBar): string|false
{
if (count($arns) === 0) {
return false;
}

// We need to set environment any value rather than "local"
// to be able to grab layers from AWS cloud.
config(['sidecar.env' => 'deploying']);

$zipArchive = new ZipArchive();

$basePath = config('sidecar-local.path') ?? $function->package()->getBasePath();
$functionLayerDirectory = "{$basePath}/../layers/{$function->name()}";

$this->filesystem->ensureDirectoryExists($functionLayerDirectory);

$progressBar->start();

foreach ($arns as $arn) {
$layerTmpFilePath = $this->getLayerTmpPath($arn);

$layerData = $this->checkArchitecture($function, $this->fetchByArn($arn));

$layerLocation = data_get($layerData, 'Content.Location');

if (! $layerLocation || ! $this->filesystem->copy($layerLocation, $layerTmpFilePath)) {
throw new \Exception("Cannot download layer for function '{$function->name()}'.");
}

$zipArchive->open($layerTmpFilePath);
$zipArchive->extractTo($functionLayerDirectory);
$zipArchive->close();

$progressBar->advance();
}

$progressBar->clear();

return Str::of($functionLayerDirectory)
->replaceFirst($basePath, '.')
->append(':/opt:ro')
->value();
}

/**
* Get temporary path for layer artifacts (files).
*/
protected function getLayerTmpPath(string $arn): string
{
$layerDirectoryName = Str::of($arn)->afterLast(':layer:')->replaceLast(':', '/');
$layerZipFileName = $layerDirectoryName->afterLast('/')->append('.zip')->value();
$layerDirectoryName = $layerDirectoryName->value();

$layerTmpDirectoryPath = sys_get_temp_dir()."/SidecarLocal/layers/{$layerDirectoryName}";

return "{$layerTmpDirectoryPath}/{$layerZipFileName}";
}

/**
* Fetch layer by ARN string.
*/
protected function fetchByArn(string $arn): array
{
return $this->client->getLayerVersionByArn(['Arn' => $arn])->toArray();
}

/**
* Check function runtime is within layer compatible runtime list.
*/
protected function checkRuntime(LambdaFunction $function, array $layer): array
{
if (! isset($layer['CompatibleRuntimes'])) {
return $layer;
}

if (! in_array($function->runtime(), $layer['CompatibleRuntimes'])) {
throw new SidecarIncompatibleFunctionRuntime(
sprintf(
SidecarIncompatibleFunctionRuntime::MESSAGE,
$function->architecture(),
$function->name(),
implode(', ', $layer['CompatibleRuntimes'])
)
);
}

return $layer;
}

/**
* Check function architecture is within supported layer architecture list.
*/
protected function checkArchitecture(LambdaFunction $function, array $layer): array
{
if (! isset($layer['CompatibleArchitectures'])) {
return $layer;
}

if (! in_array($function->architecture(), $layer['CompatibleArchitectures'])) {
throw new SidecarIncompatibleFunctionArchitecture(
sprintf(
SidecarIncompatibleFunctionArchitecture::MESSAGE,
$function->architecture(),
$function->name(),
implode(', ', $layer['CompatibleArchitectures'])
)
);
}

return $layer;
}
}
Loading

0 comments on commit 964dae7

Please sign in to comment.