refactor: update sql schema autogen tool (#331)

* chore(tooling): swap out dbviz custom database diagram tool for SchemaSpy, remove dependencies in catalyst-ci postgresql base * chore(tooling): swap out dbviz custom database diagram tool for SchemaSpy, remove dependencies in catalyst-ci postgresql base * chore(tooling): refactored how the dependencies are downloaded and cached to avoid apt install executing every run (reduced network calls) * chore(tooling): refactored how the dependencies are downloaded and cached to avoid apt install executing every run (reduced network calls) * chore(tooling): Fix markdown warnings breaking the CI build. No warnings required on md file as it is just a container for SchemaSpy * chore(tooling): fix breaking CI issues * chore(tooling): fix breaking CI issues * refactor(tooling): modified approach to align with repo goals based on PR feedback. Removed seperate layers for package installs * refactor(tooling): re-ordered package install and download steps for improved caching * refactor(tooling): removed dbviz utility from repository, dbviz is no longer required and has been replaced with SchemaSpy for database schema visualizations diagram generation --------- Co-authored-by: Steven Johnson <[email protected]>
input-output-hk · Oct 16, 2024 · 6f05e44 · 6f05e44
1 parent 9550d36
commit 6f05e44
Show file tree

Hide file tree

Showing 22 changed files with 148 additions and 1,590 deletions.
diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic
@@ -95,6 +95,7 @@ rustflags
 rustfmt
 rustup
 sanchonet
+schemaspy
 sqlfluff
 stdcfgs
 subproject
@@ -120,4 +121,5 @@ WORKDIR
 xerrors
 xvfb
 zstd
-zstdcat
+zstdcat
+JDBC
diff --git a/earthly/java/Earthfile b/earthly/java/Earthfile
@@ -0,0 +1,15 @@
+VERSION 0.8
+
+# Base image for Java used in other targets to avoid improve caching
+java-base:
+    FROM openjdk:21-jdk-slim
+
+    SAVE ARTIFACT /usr/local/openjdk-21 /java
+
+COPY_DEPS:
+    FUNCTION
+    COPY +java-base/java /usr/local/openjdk-21
+
+    # Set environment variables for Java
+    ENV JAVA_HOME=/usr/local/openjdk-21
+    ENV PATH=$JAVA_HOME/bin:$PATH
diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile
@@ -2,14 +2,17 @@
 VERSION 0.8
 
 IMPORT ../rust/tools AS rust-tools
-IMPORT ../../utilities/dbviz AS dbviz
 IMPORT ../../utilities/scripts AS scripts
+IMPORT ../java AS java
 
-# cspell: words psycopg dbviz
+# cspell: words psycopg
 
 postgres-base:
     FROM postgres:16.4-bookworm
 
+    ARG SCHEMASPY_VERSION=6.2.4
+    ARG POSTGRESQL_JDBC_VERSION=42.7.4
+
     WORKDIR /root
 
     # Install necessary packages 
@@ -22,7 +25,7 @@ postgres-base:
         colordiff \
         findutils \
         fontconfig \
-        fonts-liberation2        \
+        fonts-liberation2 \
         graphviz \
         libssl-dev \
         mold \
@@ -36,17 +39,20 @@ postgres-base:
         pipx \
         && rm -rf /var/lib/apt/lists/*
 
+    # Use the cached java installation from the java Earthfile
+    DO java+COPY_DEPS
+
     # Install SQLFluff - Check
     ENV PATH="/root/.local/bin:${PATH}"
     RUN pipx install sqlfluff==3.1.1 && sqlfluff version
 
+    # Install SchemaSpy and required Postgresql JDBC driver
+    RUN wget -O /bin/postgresql.jar https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_JDBC_VERSION}.jar
+    RUN wget -O /bin/schemaspy.jar https://github.com/schemaspy/schemaspy/releases/download/v${SCHEMASPY_VERSION}/schemaspy-${SCHEMASPY_VERSION}.jar
+
     # Get refinery
     COPY rust-tools+tool-refinery/refinery /bin
 
-    # Get dbviz
-    COPY dbviz+build/dbviz /bin
-    RUN dbviz --help
-
     # Copy our set SQL files
     COPY --dir sql /sql
 
@@ -56,7 +62,11 @@ postgres-base:
     DO scripts+ADD_BASH_SCRIPTS
     DO scripts+ADD_PYTHON_SCRIPTS
 
+    # Copy templates to the working directory
+    COPY --dir templates /templates
+
     SAVE ARTIFACT /scripts /scripts
+    SAVE ARTIFACT /templates /templates
 
 # Common build setup steps.
 # Arguments:
@@ -82,8 +92,13 @@ BUILDER:
 
 # DOCS - FUNCTION to build the docs, needs to be run INSIDE the BUILDER like so:
 # 
-# 1. Create a ./docs/diagrams.json which has the options needed to run to generate the docs to /docs
-# 2. Define the following targets in your earthfile
+# This function uses SchemaSpy to generate database documentation.
+# SchemaSpy creates detailed, Discoverable ER diagrams and schema documentation.
+#
+# To use this function:
+# 1. Ensure your migrations are in the ./migrations directory
+# 2. Have a refinery.toml file to configure the migrations
+# 3. Define the following targets in your earthfile:
 #
 # builder:
 #    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:<ver>+BUILDER --sqlfluff_cfg=./../../+repo-config/repo/.sqlfluff
@@ -93,25 +108,27 @@ BUILDER:
 #
 #    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:<ver>+BUILD --image_name=<name>
 #    DO github.com/input-output-hk/catalyst-ci/earthly/postgresql:<ver>+DOCS
+#
+# The generated documentation will be saved in the ./docs artifact.
 DOCS:
     FUNCTION
 
-    ARG diagrams=./diagrams.json
     ARG migrations=./migrations
     ARG refinery_toml=./refinery.toml
 
+    FROM +postgres-base
+
     USER postgres:postgres
     WORKDIR /docs
 
-    COPY $diagrams ./diagrams.json
     COPY --dir $migrations .
     COPY --dir $refinery_toml .
 
-    RUN /scripts/std_docs.py ./diagrams.json
-
-    SAVE ARTIFACT docs /docs
-
+    RUN /scripts/std_docs.py
+    # Pull templates artifact from postgres-base
+    COPY +postgres-base/templates/schema.md ./docs/schema.md
 
+    SAVE ARTIFACT docs ./docs
 
 # Linter checks for sql files
 CHECK:
@@ -171,3 +188,4 @@ BUILD:
     # Push the container...
     SAVE IMAGE ${image_name}:latest
 
+