Enable download using uploaded artifact

.github/workflows/publish_lambda_packages.yml

@@ -27,37 +27,35 @@ jobs:
         run: mypy distribution/lambda/
 
       - name: Extract asset version of release
-        run: echo "ASSET_VERSION=${GITHUB_REF/refs\/tags\//}-beta" >> $GITHUB_ENV
+        run: echo "QW_LAMBDA_VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
         if: ${{ github.event_name == 'push' }}
       - name: Setting version as prerelease
-        run: echo "ASSET_VERSION=${{ env.PRERELEASE_VERSION_NAME }}" >> $GITHUB_ENV
-        if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' }}
-      - name: Extract asset name
-        run: |
-          echo "SEARCHER_ASSET_FULL_NAME=quickwit-lambda-searcher-${{ env.ASSET_VERSION }}-x86_64.zip" >> $GITHUB_ENV
-          echo "INDEXER_ASSET_FULL_NAME=quickwit-lambda-indexer-${{ env.ASSET_VERSION }}-x86_64.zip" >> $GITHUB_ENV
+        run: echo "QW_LAMBDA_VERSION=${{ env.PRERELEASE_VERSION_NAME }}" >> $GITHUB_ENV
+        if: ${{ github.event_name != 'push' }}
       - name: Retrieve and export commit date, hash, and tags
         run: |
             echo "QW_COMMIT_DATE=$(TZ=UTC0 git log -1 --format=%cd --date=format-local:%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_ENV
             echo "QW_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV
             echo "QW_COMMIT_TAGS=$(git tag --points-at HEAD | tr '\n' ',')" >> $GITHUB_ENV
       - name: Build Quickwit Lambdas
-        run: |
-          cargo lambda build -p quickwit-lambda --release --output-format zip
-          mv target/lambda/searcher/bootstrap.zip ${{ env.SEARCHER_ASSET_FULL_NAME }}
-          mv target/lambda/indexer/bootstrap.zip ${{ env.INDEXER_ASSET_FULL_NAME }}
+        run: make package
         env:
           QW_COMMIT_DATE: ${{ env.QW_COMMIT_DATE }}
           QW_COMMIT_HASH: ${{ env.QW_COMMIT_HASH }}
           QW_COMMIT_TAGS: ${{ env.QW_COMMIT_TAGS }}
-        working-directory: ./quickwit
-
+          QW_LAMBDA_BUILD: 1
+        working-directory: ./distribution/lambda
+      - name: Extract package locations
+        run: |
+          echo "SEARCHER_PACKAGE_LOCATION=./distribution/lambda/$(make searcher-package-path)" >> $GITHUB_ENV
+          echo "INDEXER_PACKAGE_LOCATION=./distribution/lambda/$(make indexer-package-path)" >> $GITHUB_ENV
+        working-directory: ./distribution/lambda
       - name: Upload Lambda archives
         uses: quickwit-inc/upload-to-github-release@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          file: quickwit/${{ env.SEARCHER_ASSET_FULL_NAME }};quickwit/${{ env.INDEXER_ASSET_FULL_NAME }}
+          file: ${{ env.SEARCHER_PACKAGE_LOCATION }};${{ env.INDEXER_PACKAGE_LOCATION }}
           overwrite: true
-          draft: ${{ env.ASSET_VERSION != env.PRERELEASE_VERSION_NAME }}
-          tag_name: aws-lambda-${{ env.ASSET_VERSION }}
+          draft: ${{ env.QW_LAMBDA_VERSION != env.PRERELEASE_VERSION_NAME }}
+          tag_name: aws-lambda-${{ env.QW_LAMBDA_VERSION }}

distribution/lambda/.gitignore

@@ -14,4 +14,7 @@ cdk.out
 # AWS SAM build directory
 .aws-sam
 
+# Benchmark output files
 *.log
+
+# Benchmark output files

distribution/lambda/Makefile

@@ -3,6 +3,13 @@
 SHELL := bash
 .SHELLFLAGS := -eu -o pipefail -c
 
+QW_LAMBDA_VERSION?=local
+PACKAGE_BASE_URL=https://github.com/quickwit-oss/quickwit/releases/download/aws-lambda-$(QW_LAMBDA_VERSION)/
+SEARCHER_PACKAGE_FILE=quickwit-lambda-searcher-$(QW_LAMBDA_VERSION)-x86_64.zip
+INDEXER_PACKAGE_FILE=quickwit-lambda-indexer-$(QW_LAMBDA_VERSION)-x86_64.zip
+export SEARCHER_PACKAGE_PATH=cdk.out/$(SEARCHER_PACKAGE_FILE)
+export INDEXER_PACKAGE_PATH=cdk.out/$(INDEXER_PACKAGE_FILE)
+
 check-env:
 ifndef CDK_ACCOUNT
 	$(error CDK_ACCOUNT is undefined)
@@ -11,16 +18,43 @@ ifndef CDK_REGION
 	$(error CDK_REGION is undefined)
 endif
 
-build:
-	cargo lambda build --manifest-path=../../quickwit/Cargo.toml -p quickwit-lambda --release
-
-bootstrap: build check-env
+# Download the packages from the Quickwit release page
+# The package varsion can be set with QW_LAMBDA_VERSION
+# Build them locally by setting QW_LAMBDA_BUILD=1
+package:
+	mkdir -p cdk.out
+	if [ "$${QW_LAMBDA_BUILD:-0}" = "1" ]
+	then
+	  cargo lambda build \
+	    --manifest-path=../../quickwit/Cargo.toml \
+	    -p quickwit-lambda \
+	    --release \
+	    --output-format zip \
+		--target x86_64-unknown-linux-gnu
+	  cp -u ../../quickwit/target/lambda/searcher/bootstrap.zip $(SEARCHER_PACKAGE_PATH)
+	  cp -u ../../quickwit/target/lambda/indexer/bootstrap.zip $(INDEXER_PACKAGE_PATH)
+	else
+	  if ! [ -f $(SEARCHER_PACKAGE_PATH) ]; then
+	    curl -C - -Ls -o $(SEARCHER_PACKAGE_PATH) $(PACKAGE_BASE_URL)$(SEARCHER_PACKAGE_FILE)
+	  fi
+	  if ! [ -f $(INDEXER_PACKAGE_PATH) ]; then
+	  	curl -C - -Ls -o $(INDEXER_PACKAGE_PATH) $(PACKAGE_BASE_URL)$(INDEXER_PACKAGE_FILE)
+	  fi
+	fi
+
+indexer-package-path:
+	echo -n $(INDEXER_PACKAGE_PATH)
+
+searcher-package-path:
+	echo -n $(SEARCHER_PACKAGE_PATH)
+
+bootstrap: package check-env
 	cdk bootstrap aws://$$CDK_ACCOUNT/$$CDK_REGION
 
 generate-base-stack:
 	cdk synth -a cdk/app.py BaseQuickwitStack
 
-deploy-hdfs: build check-env
+deploy-hdfs: package check-env
 	cdk deploy -a cdk/app.py HdfsStack
 
 invoke-hdfs-indexer: check-env
@@ -33,7 +67,7 @@ invoke-hdfs-searcher-term: check-env
 invoke-hdfs-searcher-histogram: check-env
 	python -c 'from cdk import cli; cli.invoke_hdfs_searcher("""{ "query": "*", "max_hits": 0, "aggs": { "events": { "date_histogram": { "field": "timestamp", "fixed_interval": "1d" }, "aggs": { "log_level": { "terms": { "size": 10, "field": "severity_text", "order": { "_count": "desc" } } } } } } }""")'
 
-deploy-mock-data: build check-env
+deploy-mock-data: package check-env
 	cdk deploy -a cdk/app.py MockDataStack
 
 invoke-mock-data-searcher: check-env
@@ -87,3 +121,7 @@ bench-search:
 
 destroy:
 	cdk destroy -a cdk/app.py HdfsStack MockDataStack
+
+
+clean:
+	rm -rf cdk.out

distribution/lambda/cdk/stacks/services/indexer_service.py

@@ -14,14 +14,15 @@ def __init__(
         index_config_key: str,
         memory_size: int,
         environment: dict[str, str],
+        asset_path: str,
         **kwargs,
     ) -> None:
         super().__init__(scope, construct_id, **kwargs)
 
         self.lambda_function = aws_lambda.Function(
             self,
             id="Lambda",
-            code=aws_lambda.Code.from_asset("../../quickwit/target/lambda/indexer"),
+            code=aws_lambda.Code.from_asset(asset_path),
             runtime=aws_lambda.Runtime.PROVIDED_AL2,
             handler="N/A",
             environment={

distribution/lambda/cdk/stacks/services/quickwit_service.py

@@ -1,4 +1,6 @@
 import os
+from enum import Enum
+from typing import Literal
 
 import aws_cdk
 from aws_cdk import aws_s3
@@ -12,6 +14,17 @@ def extract_local_env() -> dict[str, str]:
     return {k: os.environ[k] for k in os.environ.keys() if k.startswith("QW_LAMBDA_")}
 
 
+def package_location_from_env(type: Literal["searcher"] | Literal["indexer"]) -> str:
+    path_var = f"{type.upper()}_PACKAGE_PATH"
+    if path_var in os.environ:
+        return os.environ[path_var]
+    else:
+        print(f"Could not infer the {type} package location. You must either:")
+        print(f"- configure location in the QuickwitService construct")
+        print(f"- configure the path using {path_var}")
+        exit(1)
+
+
 class QuickwitService(Construct):
     def __init__(
         self,
@@ -24,9 +37,20 @@ def __init__(
         indexer_environment: dict[str, str] = {},
         searcher_memory_size: int = 1024,
         searcher_environment: dict[str, str] = {},
-        **kwargs
+        searcher_package_location: str | None = None,
+        indexer_package_location: str | None = None,
+        **kwargs,
     ) -> None:
+        """Create a new Quickwit Lambda service construct node.
+
+        If `{indexer|searcher}_package_location` is `None`, its value is inferred
+        from the environment variable `{INDEXER|SEARCHER}_PACKAGE_PATH`.
+        """
         super().__init__(scope, construct_id, **kwargs)
+        if searcher_package_location is None:
+            searcher_package_location = package_location_from_env("searcher")
+        if indexer_package_location is None:
+            indexer_package_location = package_location_from_env("indexer")
         self.bucket = aws_s3.Bucket(
             self,
             "IndexStore",
@@ -41,6 +65,7 @@ def __init__(
             index_config_key=index_config_key,
             memory_size=indexer_memory_size,
             environment=indexer_environment,
+            asset_path=indexer_package_location,
         )
         self.searcher = searcher_service.SearcherService(
             self,
@@ -49,4 +74,5 @@ def __init__(
             index_id=index_id,
             memory_size=searcher_memory_size,
             environment=searcher_environment,
+            asset_path=searcher_package_location,
         )

distribution/lambda/cdk/stacks/services/searcher_service.py

@@ -12,14 +12,15 @@ def __init__(
         index_id: str,
         memory_size: int,
         environment: dict[str, str],
+        asset_path: str,
         **kwargs
     ) -> None:
         super().__init__(scope, construct_id, **kwargs)
 
         self.lambda_function = aws_lambda.Function(
             self,
             id="Lambda",
-            code=aws_lambda.Code.from_asset("../../quickwit/target/lambda/searcher"),
+            code=aws_lambda.Code.from_asset(asset_path),
             runtime=aws_lambda.Runtime.PROVIDED_AL2,
             handler="N/A",
             environment={