Add configuration for EC2 deploys (#380)

.github/workflows/perform-action.yml

@@ -0,0 +1,65 @@
+name: Perform Action
+run-name: ${{ inputs.action }} by @${{ github.actor }}
+
+on:
+  workflow_call:
+    secrets:
+      TERRAFORM_AWS_ACCESS_KEY_ID:
+        required: false # Not needed for image builds
+      TERRAFORM_AWS_SECRET_ACCESS_KEY:
+        required: false # Not needed for image builds
+      STACK_DEPLOY_SSH_PRIVATE_KEY:
+        required: true
+      DOCKERHUB_USER:
+        required: true
+      DOCKERHUB_PASSWORD:
+        required: true
+    inputs:
+      action:
+        description: Action to perform
+        required: true
+        type: string
+
+  workflow_dispatch:
+    inputs:
+      action:
+        description: Action to perform
+        required: true
+        type: choice
+        options:
+          - build
+          - plan
+          - deploy
+
+jobs:
+  action:
+    runs-on: [earthly-satellite#backend-runner]
+    name: ${{ inputs.action }}
+    steps:
+      - name: Clone repo
+        uses: actions/checkout@v4
+
+      - name: Set release ID
+        id: set-release
+        run: |
+          release=$(date -u +"%Y-%m-%dT%H-%M-%S-%3NZ")
+          echo "release = $release"
+          echo "release=$release" >> "$GITHUB_OUTPUT"
+
+      - name: Set commit hash
+        id: set-commit-hash
+        run: |
+          commit_hash=$(git rev-parse HEAD)
+          echo "commit_hash = $commit_hash"
+          echo "commit_hash=$commit_hash" >> "$GITHUB_OUTPUT"
+
+      - name: Execute ${{ inputs.action }}
+        uses: ./.github/actions/command
+        with:
+          aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }}
+          ssh-private-key: ${{ secrets.STACK_DEPLOY_SSH_PRIVATE_KEY }}
+          dockerhub-username: ${{ secrets.DOCKERHUB_USER }}
+          dockerhub-password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          command: make ${{ inputs.action }} ref=${{ steps.set-commit-hash.outputs.commit_hash }} release=${{ steps.set-release.outputs.release }}
+          healthcheck: curl --fail https://fnames.farcaster.xyz/_health

Earthfile

@@ -0,0 +1,49 @@
+VERSION --run-with-aws 0.8
+PROJECT farcasterxyz/fname-registry
+FROM alpine:3.20.3
+WORKDIR /workspace
+
+ARG --global --required fname_registry_commit_ref
+ARG --global
+docker_registry=526236635984.dkr.ecr.us-east-1.amazonaws.com/farcasterxyz/fname-registry
+ARG --global FNAME_REGISTRY_DOCKER_IMAGE=$docker_registry:$fname_registry_commit_ref
+
+fname-registry-repo:
+  COPY . .
+  SAVE ARTIFACT /workspace /workspace
+
+fname-registry-prod:
+  FROM DOCKERFILE -f +fname-registry-repo/workspace/Dockerfile +fname-registry-repo/workspace/*
+  SAVE IMAGE --push $FNAME_REGISTRY_DOCKER_IMAGE
+
+stack-repo:
+  ARG stack_repo_git_url=[email protected]:warpcast/stack # Must use SSH since private
+  ARG stack_repo_commit_ref=main
+  GIT CLONE --branch $stack_repo_commit_ref $stack_repo_git_url /repo
+  SAVE ARTIFACT /repo /repo
+
+workspace:
+  FROM DOCKERFILE -f +stack-repo/repo/Dockerfile +stack-repo/repo/*
+  CACHE --sharing locked --persist /usr/src/app/workspace/cdktf.out
+  COPY +fname-registry-repo/workspace /usr/src/app/workspace
+  ARG CI
+  ENV CI=$CI
+  ENV FNAME_REGISTRY_DOCKER_IMAGE=$FNAME_REGISTRY_DOCKER_IMAGE
+  # OTEL is enabled by default by Earthly but causes crashes. Disable since we don't use
+  # See https://github.com/earthly/earthly/issues/4260
+  ENV OTEL_METRICS_EXPORTER=none
+  ENV OTEL_TRACES_EXPORTER=none
+
+interactive-cmd:
+  FROM +workspace
+  ARG args
+  RUN --interactive --no-cache --ssh \
+    --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \
+    bun cli.js --workdir ./workspace $args
+
+cmd:
+  FROM +workspace
+  ARG args
+  RUN --no-cache --ssh \
+    --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \
+    bun cli.js --workdir ./workspace $args

Makefile

@@ -0,0 +1,69 @@
+# Lists `make <cmd>` shortcuts which are easier to type than the full command.
+
+ifeq ($(CI), true)
+	EARTHLY_INTERACTIVE_ARGS := --ci
+	CMD_TYPE := cmd
+	CI_ARGS := --CI=true
+else
+	EARTHLY_INTERACTIVE_ARGS := --interactive
+	CMD_TYPE := interactive-cmd
+	CI_ARGS :=
+endif
+
+ifeq ($(release),)
+	release=
+	RELEASE_ARGS := --release=$(shell node -e "console.log(new Date().toISOString().replace(/\:/g,'-').replace(/\./g,'-'))")
+endif
+
+REF_ARGS := --fname_registry_commit_ref=$(shell git rev-parse HEAD)
+
+ifdef pod
+	POD_ARGS := pod:$(pod)
+else
+	POD_ARGS :=
+endif
+
+CMD_PREFIX := earthly --env-file-path /dev/null --max-remote-cache
+
+# Builds (but doesn't publish) the current Docker image specified by fname_registry_commit_ref in the Earthfile and outputs it locally.
+.PHONY: build
+build:
+	$(CMD_PREFIX) $(EARTHLY_INTERACTIVE_ARGS) +fname-registry-prod $(REF_ARGS) $(CI_ARGS)
+
+# Builds and publishes the current Docker image specified by fname_registry_commit_ref in the Earthfile
+.PHONY: publish
+publish:
+	$(CMD_PREFIX) --push +fname-registry-prod $(REF_ARGS)
+
+# Shows what infrastructure changes will be applied on the next deploy, if any.
+.PHONY: plan
+plan:
+	$(CMD_PREFIX) --ci --no-output +cmd $(RELEASE_ARGS) $(REF_ARGS) --CI=true --args="plan $(POD_ARGS)"
+
+# Applies any infrastructure changes without carrying out the rest of the deploy
+# process (note: this could still result in a "deploy" if updating ASGs).
+.PHONY: apply
+apply: publish
+	$(CMD_PREFIX) $(EARTHLY_INTERACTIVE_ARGS) --no-output +$(CMD_TYPE) $(RELEASE_ARGS) $(REF_ARGS) $(CI_ARGS) --args="deploy --apply-only --yes $(POD_ARGS)"
+
+# Applies any infrastructure changes (if any) and carries out a deploy.
+.PHONY: deploy
+deploy: publish
+	$(CMD_PREFIX) $(EARTHLY_INTERACTIVE_ARGS) --no-output +$(CMD_TYPE) $(RELEASE_ARGS) $(REF_ARGS) $(CI_ARGS) --args="deploy --yes $(POD_ARGS)"
+
+# DANGEROUS. Deletes infrastructure.
+.PHONY: destroy
+destroy:
+	$(CMD_PREFIX) $(EARTHLY_INTERACTIVE_ARGS) --no-output +$(CMD_TYPE) $(RELEASE_ARGS) $(REF_ARGS) $(CI_ARGS) --args="destroy --yes $(POD_ARGS)"
+
+# Validates configuration files, reporting any issues.
+.PHONY: lint
+lintit:
+	$(CMD_PREFIX) $(EARTHLY_INTERACTIVE_ARGS) --no-output +$(CMD_TYPE) $(RELEASE_ARGS) $(REF_ARGS) $(CI_ARGS) --args="lint"
+
+# Open an SSH console to the specified pod.
+#
+# e.g. `make ssh pod=mypod`
+.PHONY: ssh
+ssh:
+	$(CMD_PREFIX) --no-output +$(CMD_TYPE) $(RELEASE_ARGS) $(REF_ARGS) --args="console $(POD_ARGS)"

deploy-docker-compose.yml

@@ -0,0 +1,134 @@
+# Docker Compose configuration shared by all pods for the backend.
+#
+# If you need to tweak something for a specific pod, reference it as an
+# environment variable here and set that environment variable explicitly for
+# each pod in deploy.yml
+
+x-shared-labels: &shared-labels
+  com.datadoghq.tags.pod: ${POD_NAME}
+  com.datadoghq.tags.service: fname-registry-${POD_NAME}
+
+# Blank values implies values are inherited from the environment
+x-shared-env-vars: &shared-env-vars
+  TINI_VERBOSITY: 3
+  ENVIRONMENT: prod
+  FC_NETWORK_ID: "1"
+  WARPCAST_ADDRESS: "0xABba722926c8302c73e57A25AD8F63753904546f"
+  CCIP_ADDRESS: "0x145b9934B42F214C101De04b6115285959BDD4F5"
+  DD_API_KEY:
+  MNEMONIC:
+  OP_ALCHEMY_SECRET:
+  MAINNET_ALCHEMY_SECRET:
+  ETHERSCAN_API_SECRET:
+  INFURA_PROJECT_ID:
+  INFURA_PROJECT_SECRET:
+  POSTGRES_URL:
+  POSTGRES_URL_READ:
+  # Datadog SDK configuration. Agent configuration is further below
+  DD_API_KEY:
+  DD_TRACE_AGENT_URL: unix:///var/run/datadog/apm.socket
+  DD_RUNTIME_METRICS_ENABLED: "true"
+  DD_PROFILING_ENABLED: "true"
+  DD_LOGS_INJECTION: "true"
+  DD_ENV: prod
+  DD_VERSION: ${RELEASE}
+  DD_TRACE_DISABLED_PLUGINS: child_process, connect, dns, net
+
+services:
+  app:
+    depends_on:
+      - datadog # Make sure agent is running so all stats+logs are collected
+    image: ${FNAME_REGISTRY_DOCKER_IMAGE:-dummy-value-for-linting}
+    init: true
+    command: ["node", "index.js"]
+    environment:
+      <<: *shared-env-vars
+      TASK_TYPE: ${TASK_TYPE}
+      DD_SERVICE: fname-registry-${POD_NAME}
+      POSTGRES_CONNECTION_POOL_MAX_SIZE: "20"
+    labels:
+      <<: *shared-labels
+    network_mode: host
+    restart: always
+    volumes:
+      - /var/run/datadog/:/var/run/datadog/ # Unix socket
+    ulimits:
+      nofile:
+        soft: 65535
+        hard: 65535
+    healthcheck:
+      test: ["CMD-SHELL", "${HEALTHCHECK_CMD}"]
+      start_period: 15s # Only affects whether Docker determines the container is healthy, does not stop container if unhealthy
+      interval: 10s
+      timeout: 10s
+      retries: 3
+
+  datadog:
+    image: public.ecr.aws/datadog/agent:7.57.2
+    cgroup: host
+    pid: host
+    network_mode: host
+    restart: always
+    environment:
+      DD_LOG_LEVEL: "WARN" # For agent's logs only
+      DD_ENV: prod
+      DD_SERVICE: backend-${POD_NAME}
+      DD_VERSION: ${RELEASE}
+      DD_API_KEY: ${DD_API_KEY}
+      DD_HOSTNAME: ${INSTANCE_ID}.merkle.zone
+      DD_APM_ENABLED: "true"
+      DD_APM_RECEIVER_SOCKET: /var/run/datadog/apm.socket
+      DD_APM_NON_LOCAL_TRAFFIC: "true"
+      DD_LOGS_ENABLED: "true"
+      DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: "true"
+      DD_PROCESS_CONFIG_PROCESS_COLLECTION_ENABLED: "true"
+      DD_PROCESS_AGENT_ENABLED: "true"
+      DD_SYSTEM_PROBE_NETWORK_ENABLED: "true"
+      DD_DOGSTATSD_TAG_CARDINALITY: "low"
+      DD_DOGSTATSD_NON_LOCAL_TRAFFIC: "true"
+      DD_DOGSTATSD_ORIGIN_DETECTION: "true"
+      DD_DOGSTATSD_SOCKET: /var/run/datadog/dsd.socket
+      DD_CONTAINER_LABELS_AS_TAGS: '{
+        "com.docker.compose.service": "container",
+        "com.datadoghq.tags.pod": "pod"
+      }'
+      DD_HEALTH_PORT: 5555
+    cap_add:
+      - SYS_ADMIN
+      - SYS_RESOURCE
+      - SYS_PTRACE
+      - NET_ADMIN
+      - NET_BROADCAST
+      - NET_RAW
+      - IPC_LOCK
+      - CHOWN
+    security_opt:
+      - apparmor:unconfined
+    labels:
+      <<: *shared-labels
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /proc/:/host/proc/:ro
+      - /opt/datadog-agent/run:/opt/datadog-agent/run:rw
+      - /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
+      - /etc/passwd:/etc/passwd:ro
+      - /var/run/datadog/:/var/run/datadog/
+      - /sys/kernel/debug:/sys/kernel/debug
+      - /etc/group:/etc/group:ro
+      - /usr/lib/os-release:/usr/lib/os-release:ro
+      - /sys/kernel/security:/host/sys/kernel/security:ro
+
+  # Watches container health status and restarts containers if failed
+  # (Docker doesn't do this out of the box, it only restarts if process exits)
+  autoheal:
+    environment:
+      AUTOHEAL_CONTAINER_LABEL: all
+      AUTOHEAL_START_PERIOD: 120 # Give service time to start up (2 mins)
+    image: willfarrell/autoheal:1.1.0
+    network_mode: none
+    restart: always
+    stop_signal: SIGKILL # autoheal doesn't handle signals correctly, so just kill it
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+      - /var/run/docker.sock:/var/run/docker.sock