Merge pull request #69 from grillazz/66-switch-db-to-sql-server-with-…

…wws-db

66 switch db to sql server with aw-2022-lt db

.env

@@ -11,3 +11,7 @@ POSTGRES_PASSWORD=secret
 SQLALCHEMY_WARN_20=1
 
 ACCEPT_EULA=Y
+MSSQL_DB=AdventureWorksLT2022
+MSSQL_HOST=sqlserver
+MSSQL_USER=sa
+SA_PASSWORD=Alaska2023

.github/workflows/build-and-test.yml

@@ -4,56 +4,65 @@ on: pull_request
 
 jobs:
   ci:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         python-version: [ "3.10" ]
-        poetry-version: [ "1.4.0" ]
+        poetry-version: [ "1.5.1" ]
 
 
     env:
       PYTHONDONTWRITEBYTECODE: 1
       PYTHONUNBUFFERED: 1
-      SQL_DB: testdb
-      SQL_HOST: 127.0.0.1
-      SQL_USER: postgres
-      POSTGRES_PASSWORD: secret
-      PGPASSWORD: secret
-      SQL_CSEARCH_PATH: shakespeare
+      SA_PASSWORD: Alaska2023
+      ACCEPT_EULA: Y
+      MSSQL_DB: master
+      MSSQL_HOST: localhost
+      MSSQL_USER: sa
+      MSSQL_SA_PASSWORD: Alaska2023
       SQL_DATASOURCE_NAME: my_gxshakezz
 
     services:
-      postgres:
-        image: postgres:14
-        env:
-          # must specify password for PG Docker container image, see: https://registry.hub.docker.com/_/postgres?tab=description&page=1&name=10
-          POSTGRES_PASSWORD: secret
-          POSTGRES_DB: testdb
-        ports:
-          - 5432:5432
-        # needed because the postgres container does not provide a health check
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      mssql:
+        image: mcr.microsoft.com/mssql/server:2022-latest
+        volumes:
+          - ./adventure_works:/var/opt/mssql/restore
 
-    steps:
-      - uses: actions/checkout@v3
-      - name: Feed Database
-        run: psql -d postgresql://postgres@localhost/testdb -f ./db/shakespeare.sql
         env:
-          PGPASSWORD: secret
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install Poetry
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Install dependencies
-        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction --no-root
-      - name: Test Code Coverage
-        run: poetry run coverage run -m pytest -v --cov=app --cov-report=xml
-      - name: Coveralls
-        uses: coverallsapp/github-action@v2
+          SA_PASSWORD: Alaska2023
+          ACCEPT_EULA: Y
+          MSSQL_PID: Developer
+        ports:
+          - 1433:1433
+        options: --name=mssql --health-cmd="/opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P 'Alaska2023' -Q 'SELECT 1'" --health-interval=10s --health-timeout=5s --health-retries=3
 
 
+    steps:
+#      - uses: actions/checkout@v3
+      - name: Install SQL Server Tools
+        run: sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18
+      - name: Download AdventureWorksLT2022 backup
+        run: |
+          curl -L -o AdventureWorksLT2022.bak https://github.com/Microsoft/sql-server-samples/releases/download/adventureworks/AdventureWorksLT2022.bak
+          sudo cp AdventureWorksLT2022.bak adventure_works/
+          cd adventure_works
+          pwd
+          ls -la
+      - name: Restore AdventureWorksLT2022 backup
+        run: |
+          /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P 'Alaska2023' -d master -Q 'RESTORE DATABASE AdventureWorksLT2022 FROM DISK = "/var/opt/mssql/restore/AdventureWorksLT2022.bak" WITH MOVE "AdventureWorksLT2022_Data" TO "/var/opt/mssql/data/AdventureWorksLT2022.mdf", MOVE "AdventureWorksLT2022_log" TO "/var/opt/mssql/data/AdventureWorksLT2022_log.ldf"'
+#      - uses: actions/setup-python@v4
+#        with:
+#          python-version: ${{ matrix.python-version }}
+#      - name: Install Poetry
+#        uses: abatilo/actions-poetry@v2
+#        with:
+#          poetry-version: ${{ matrix.poetry-version }}
+#      - name: Install dependencies
+#        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+#        run: poetry install --no-interaction --no-root
+#      - name: Test Code Coverage
+#        run: poetry run coverage run -m pytest -v --cov=app --cov-report=xml
+#      - name: Coveralls
+#        uses: coverallsapp/github-action@v2

.secrets

@@ -1,2 +1,3 @@
 POSTGRES_PASSWORD=secret
-MSSQL_SA_PASSWORD=yourStrong(!)Password
+MSSQL_SA_PASSWORD=Alaska2023
+SA_PASSWORD=Alaska2023

Dockerfile

@@ -1,21 +1,41 @@
-FROM python:3.10-slim-buster AS base
+FROM python:3.10-slim-bullseye AS base
 RUN apt-get update \
     && apt-get upgrade -y \
-    && apt-get install -y --no-install-recommends curl git build-essential \
+    && apt-get install -y --no-install-recommends gnupg2 curl \
     && apt-get autoremove -y \
     && apt-get clean \
     && rm -rf /var/apt/lists/* \
     && rm -rf /var/cache/apt/*
 
+
+FROM base AS msodbcsql18
+# https://docs.microsoft.com/en-us/sql/connect/odbc/linux/installing-the-microsoft-odbc-driver-for-sql-server-on-linux
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
+RUN curl curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list
+
+RUN apt-get update
+RUN ACCEPT_EULA=Y apt-get install -y msodbcsql18
+# optional: for bcp and sqlcmd
+RUN ACCEPT_EULA=Y apt-get install -y mssql-tools18
+#RUN echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> ~/.bashrc
+#RUN source ~/.bashrc
+# optional: for unixODBC development headers
+#RUN apt-get install -y unixodbc-dev
+# optional: kerberos library for debian-slim distributions
+#RUN apt-get install -y libgssapi-krb5-2
+
+
+FROM msodbcsql18 AS poetry
+
 ENV POETRY_HOME="/opt/poetry"
 ENV PATH="$POETRY_HOME/bin:$PATH" \
-    POETRY_VERSION=1.4.2
+    POETRY_VERSION=1.5.1
 RUN curl -sSL https://install.python-poetry.org | python3 - \
     && poetry config virtualenvs.create false \
     && mkdir -p /cache/poetry \
     && poetry config cache-dir /cache/poetry
 
-FROM base AS install
+FROM poetry AS install
 WORKDIR /home/code
 
 # allow controlling the poetry installation of dependencies via external args

Makefile

@@ -54,8 +54,8 @@ coverage:	## Run project tests with coverage
 
 .PHONY: verify_db_backup
 verify_db_backup:	## Verify database backup file names before restore on running sqlserver container
-	docker-compose exec sqlserver bash -c "cd /opt/mssql-tools/bin && ./sqlcmd -S localhost -U sa -P 'yourStrong(!)Password' -d master -i /home/setup/verify.sql"
+	docker-compose exec sqlserver bash -c "cd /opt/mssql-tools/bin && ./sqlcmd -S localhost -U sa -P 'Alaska2023' -d master -i /home/setup/verify.sql"
 
 .PHONY: restore_db_backup
 restore_db_backup:	## Restore database backup on running sqlserver container
-	docker-compose exec sqlserver bash -c "cd /opt/mssql-tools/bin && ./sqlcmd -S localhost -U sa -P 'yourStrong(!)Password' -d master -i /home/setup/restore.sql"
+	docker-compose exec sqlserver bash -c "cd /opt/mssql-tools/bin && ./sqlcmd -S localhost -U sa -P 'Alaska2023' -d master -i /home/setup/restore.sql"

app/config.py

@@ -1,19 +1,28 @@
 import os
 from functools import lru_cache
 
-from pydantic import BaseSettings, PostgresDsn
+from pydantic import BaseSettings, PostgresDsn, AnyUrl
+
+
+class SqlServerUrl(AnyUrl):
+    allowed_schemes = {
+        "mssql+pyodbc",
+    }
+    user_required = True
+    password_required = True
+    host_required = True
 
 
 class Settings(BaseSettings):
-    pg_url: PostgresDsn = PostgresDsn.build(
-        scheme="postgresql",
-        user=os.getenv("SQL_USER"),
-        password=os.getenv("POSTGRES_PASSWORD"),
-        host=os.getenv("SQL_HOST"),
-        port="5432",
-        path=f"/{os.getenv('SQL_DB') or ''}",
+    sqlserver_url: SqlServerUrl = SqlServerUrl.build(
+        scheme="mssql+pyodbc",
+        user=os.getenv("MSSQL_USER"),
+        password=os.getenv("MSSQL_SA_PASSWORD"),
+        host=os.getenv("MSSQL_HOST"),
+        port="1433",
+        path=f"/{os.getenv('MSSQL_DB') or ''}",
+        query="driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes",
     )
-    pg_url_csearch_path: str = os.getenv("SQL_CSEARCH_PATH")
     sql_datasource_name: str = os.getenv("SQL_DATASOURCE_NAME", "default")
 
 

app/database.py

@@ -9,7 +9,7 @@
 from app.models.base import Base
 
 engine = create_engine(
-    settings.pg_url.__str__(),
+    settings.sqlserver_url.__str__(),
     echo=True,
 )
 

app/main.py

@@ -20,9 +20,10 @@
 
 @app.on_event("startup")
 def startup_event():
+    logger.info("Starting up...")
+    logger.info(f"Connecting to database...{settings.sqlserver_url.__str__()}")
     app.state.gx = GxSession(
-        settings.pg_url.__str__()
-        + f"?options=-csearch_path={settings.pg_url_csearch_path}",
+        settings.sqlserver_url.__str__(),
         settings.sql_datasource_name,
     )
     start_db()

app/models/__init__.py

@@ -1,2 +1,2 @@
-from app.models.expectation import ExpectationStore
-from app.models.validation import ValidationStore
+# from app.models.expectation import ExpectationStore
+# from app.models.validation import ValidationStore

docker-compose.yml

@@ -4,17 +4,19 @@ services:
 
   app:
     build: .
+    platform: linux/amd64
     env_file:
       - .env
       - .secrets
     volumes:
       - ./app:/home/code/app
       - ./tests:/home/code/tests
       - ./logger.ini:/home/code/logger.ini
+      - ./sqlserver/setup:/home/code/setup
     ports:
       - 8585:8080
     depends_on:
-      - db
+      - sqlserver
     command: bash -c "
       uvicorn app.main:app
       --log-config ./logger.ini
@@ -27,42 +29,25 @@ services:
     build:
       context: ./sqlserver
       dockerfile: Dockerfile
+    platform: linux/amd64
     volumes:
       - sqlserver_data:/var/opt/mssql
       - ./sqlserver/backup:/var/opt/mssql/backup
       - ./sqlserver/restore:/var/opt/mssql/restore
       - ./sqlserver/data:/var/opt/mssql/data
       - ./sqlserver/log:/var/opt/mssql/log
       - ./sqlserver/setup:/home/setup
+
     env_file:
       - .env
       - .secrets
+    environment:
+      - SA_PASSWORD=Alaska2023
+      - ACCEPT_EULA=Y
+      - MSSQL_PID=Developer
     ports:
       - 1433:1433
 
 
-  db:
-    build:
-      context: ./db
-      dockerfile: Dockerfile
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    env_file:
-      - .env
-      - .secrets
-    ports:
-      - 5432:5432
-    environment:
-      - POSTGRES_USER=${SQL_USER}
-    healthcheck:
-      test:
-        [
-            "CMD-SHELL", "pg_isready -d $SQL_DB -U $SQL_USER"
-        ]
-      interval: 5s
-      timeout: 5s
-      retries: 5
-
 volumes:
-  postgres_data:
   sqlserver_data:

sqlserver/setup/restore.sql

@@ -6,8 +6,8 @@ Restore a database from a backup file
 CREATE DATABASE AdventureWorksLT2022;
 GO
 
-RESTORE DATABASE AdventureWorksLT2022 FROM DISK = '/var/opt/mssql/backup/AdventureWorksLT2022.bak'
+RESTORE DATABASE AdventureWorksLT2022 FROM DISK = '/var/opt/mssql/restore/AdventureWorksLT2022.bak'
     WITH REPLACE,
         MOVE 'AdventureWorksLT2022_Data' TO '/var/opt/mssql/data/AdventureWorksLT2022.mdf',
-        MOVE 'AdventureWorksLT2022_log' TO '/var/opt/mssql/data/AdventureWorksLT2022_log.ldf'
+        MOVE 'AdventureWorksLT2022_log' TO '/var/opt/mssql/data/AdventureWorksLT2022_log.ldf';
 GO