feat: Add build logic for Dagster container images

As part of the updated build/deployment we want to have separate images for the Dagit
and dagaster-daemon processes, which are also separate from the user pipeline code so
that they can all be built, deployed, and scaled independently. For the user pipelines
we also want to ensure that the dbt project is available in the runtime
environment. This does the following:
- Copy all files related to the dbt project into user pipeline images by default
- Create a multi-stage build for Dagit/dagster-daemon to avoid duplicate logic
- Moves the Dagster-specific workspace and Dagster yaml files into the `ol_orchestrate` directory
- Moves the dbt project files to the proper directory level in the repo
- Adds the initial work to package up collections of Dagster pipelines based on the 'repository' as the entry-point for the Python distribution

.pre-commit-config.yaml

@@ -3,7 +3,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.2.0
+  rev: v4.3.0
   hooks:
   - id: trailing-whitespace
   - id: end-of-file-fixer
@@ -32,7 +32,7 @@ repos:
   - id: yamllint
     args: [--format, parsable, -d, relaxed]
 - repo: https://github.com/asottile/pyupgrade
-  rev: v2.32.1
+  rev: v2.34.0
   hooks:
   - id: pyupgrade
     args:
@@ -78,7 +78,7 @@ repos:
     - --extend-ignore=D1
     - --diff
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.950
+  rev: v0.961
   hooks:
   - id: mypy
     additional_dependencies:
@@ -87,7 +87,7 @@ repos:
     - types-pytz
     - types-pymysql
 - repo: https://github.com/sqlfluff/sqlfluff
-  rev: 0.13.1
+  rev: 0.13.2
   hooks:
   - id: sqlfluff-fix
       # Arbitrary arguments to show an example

dockerfiles/BUILD

@@ -1,9 +1,29 @@
 docker_image(
-    name="docker",
+    name="dagit",
     source="Dockerfile.dagit",
+    dependencies=[
+        "src/ol_orchestrate:dagit",
+        "src/ol_orchestrate:dagster-daemon",
+        "src/ol_orchestrate:project-config",
+    ],
+    target_stage="dagit"
 )
 
 docker_image(
-    name="docker0",
+    name="dagster-daemon",
+    source="Dockerfile.dagit",
+    dependencies=[
+        "src/ol_orchestrate:dagster-daemon",
+        "src/ol_orchestrate:project-config",
+    ],
+    target_stage="dagster-daemon"
+)
+
+docker_image(
+    name="pipeline",
+    dependencies=[
+        "src/ol_orchestrate:open-edx",
+        "src/ol_dbt:dbt_project"
+    ],
     source="Dockerfile.user_pipeline",
 )

dockerfiles/Dockerfile.dagit

@@ -1,22 +1,24 @@
 FROM python:3.9-slim AS dagster-base
-RUN mkdir -p /opt/dagster/dagster_home /opt/dagster/app
-RUN useradd -s /bin/bash -d /opt/dagster/dagster_home/ dagster
-RUN chown -R dagster: /opt/dagster/
+RUN mkdir -p /opt/dagster/dagster_home /opt/dagster/app && \
+    useradd -s /bin/bash -d /opt/dagster/dagster_home/ dagster &&\
+    chown -R dagster: /opt/dagster/
 USER dagster
 ENV DAGSTER_HOME=/opt/dagster/dagster_home/
 
 # Copy your code and workspace to /opt/dagster/app
-COPY workspace.yaml /opt/dagster/app/
+COPY --chown=dagster:dagster src/ol_orchestrate/workspace.yaml /opt/dagster/app/
 
 # Copy dagster instance YAML to $DAGSTER_HOME
-COPY dagster.yaml /opt/dagster/dagster_home/
+COPY --chown=dagster:dagster src/ol_orchestrate/dagster.yaml /opt/dagster/dagster_home/
 
 WORKDIR /opt/dagster/app
 
 EXPOSE 3000
 
-FROM dagster-base AS dagit
-ENTRYPOINT ["dagit", "-h", "0.0.0.0", "-p", "3000"]
-
 FROM dagster-base AS dagster-daemon
+COPY --chown=dagster:dagster src.ol_orchestrate/dagster-daemon.pex /usr/local/bin/dagster-daemon
 ENTRYPOINT ["dagster-daemon", "run"]
+
+FROM dagster-base AS dagit
+COPY --chown=dagster:dagster src.ol_orchestrate/dagit.pex /usr/local/bin/dagit
+ENTRYPOINT ["dagit", "-h", "0.0.0.0", "-p", "3000"]

dockerfiles/Dockerfile.user_pipeline

@@ -1,3 +1,9 @@
 FROM python:3.9-slim
 
-CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", ]
+RUN mkdir -p /opt/dagster/dagster_home /opt/dagster/app /tmp/packages && \
+    useradd -s /bin/bash -d /opt/dagster/dagster_home/ dagster
+COPY *.whl /tmp/packages/
+RUN pip install --no-cache-dir /tmp/packages/* && rm -r /tmp/packages/
+COPY --chown=dagster:dagster src/ol_dbt/ /opt/dbt/
+USER dagster
+CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000"]

pyproject.toml

@@ -35,6 +35,7 @@ httpx = "^0.22.0"
 pyarrow = "^8.0.0"
 pyathena = "^2.8.0"
 pymysql = "^1.0.0"
+hvac = "^0.11.2"
 
 [tool.poetry.dev-dependencies]
 black = "*"
@@ -58,7 +59,7 @@ requires = ["poetry_core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.sqlfluff.core]
-templater = "dbt"
+templater = "jinja"
 dialect = "hive"
 sql_file_exts = ".sql,.sql.j2,.dml,.ddl"
 
@@ -67,3 +68,6 @@ unwrap_wrapped_queries = true
 
 [tool.sqlfluff.templater.jinja]
 apply_dbt_builtins = true
+
+[tool.sqlfluff.templater.dbt]
+project_dir = "src/ol_dbt/"

src/ol_dbt/BUILD

@@ -0,0 +1,4 @@
+files(
+    name="dbt_project",
+    sources=["**/*.yml", "**/*.json", "**/*.sql", "**/*.yaml"],
+)

src/ol_orchestrate/BUILD

@@ -20,22 +20,9 @@ pex_binary(
     entry_point="dagster.daemon.cli:main"
 )
 
-docker_image(
-    name="ol-dagit",
-    description="Dagster web service container iamge",
-    dependencies=[
-        ":dagit"
-    ],
-    source="dockerfiles/Dockerfile.dagster"
-)
-
-docker_image(
-    name="ol-dagster-daemon",
-    description="Dagster daemon container image for scheduling and run triggering",
-    dependencies=[
-        ":dagster-daemon"
-    ],
-    source="dockerfiles/Dockerfile.dagster"
+files(
+    name="project-config",
+    sources=["dagster.yaml", "workspace.yaml"]
 )
 
 python_distribution(

src/ol_orchestrate/dagster.yaml

@@ -0,0 +1,40 @@
+---
+postgres_config: &postgres_config
+  config:
+    postgres_db:
+      username:
+        env: DAGSTER_PG_USERNAME
+      password:
+        env: DAGSTER_PG_PASSWORD
+      hostname:
+        env: DAGSTER_PG_HOST
+      db_name:
+        env: dagster
+      port: 5432
+
+telemetry:
+  enabled: true
+scheduler:
+  module: dagster.core.scheduler
+  class: DagsterDaemonScheduler
+compute_logs:
+  module: dagster_aws.s3.compute_log_manager
+  class: S3ComputeLogManager
+  config:
+    bucket: dagster-{{ environment }}
+    prefix: compute-logs/
+run_storage:
+  module: dagster_postgres.run_storage
+  class: PostgresRunStorage
+  config:
+    <<: *postgres_config
+event_log_storage:
+  module: dagster_postgres.event_log
+  class: PostgresEventLogStorage
+  config:
+    <<: *postgres_config
+schedule_storage:
+  module: dagster_postgres.schedule_storage
+  class: PostgresScheduleStorage
+  config:
+    <<: *postgres_config