encrypt donation, separate table for newsletter emails

docker-compose.yml

@@ -19,6 +19,7 @@ services:
     restart: always
     env_file:
       - db.env
+    command: --pid-file /var/lib/mysql/mysqld.pid
     ports:
       - '3306:3306'
     expose:

requirements.txt

@@ -1,6 +1,8 @@
 flask
 flask-sqlalchemy
 sqlalchemy
+sqlalchemy-utils
+cryptography
 mysqlclient
 gunicorn
 pytest

src/app/website/__init__.py

@@ -19,7 +19,6 @@ def create_app():
     # reads the key from FLASK_SECRET_KEY env var
     # app.config['SQLALCHEMY_DATABASE_URI'] = f'mysql://donor:{PASSWD}@localhost/{DB_NAME}'
     app.config['SQLALCHEMY_DATABASE_URI'] = f'mysql://donor:{PASSWD}@127.0.0.1/{DB_NAME}'
-    print(app.config)
     db.init_app(app)
 
     from .views import views

src/app/website/donate.py

@@ -1,11 +1,11 @@
 from flask import Blueprint, render_template, request, flash, redirect, url_for
 from werkzeug.security import generate_password_hash
+from sqlalchemy import func, cast, VARBINARY
 from .models import RawData
 from . import db
 
 donate = Blueprint("donate", __name__)
 
-
 @donate.route("/donation", methods=["GET", "POST"])
 def donation():
     if request.method == "GET":
@@ -18,14 +18,18 @@ def donation():
         else:
             # at the moment we are generating the hash checksum for the raw text
             new_submission = RawData(
-                donation=text,
-                checksum=generate_password_hash(text, method="pbkdf2:sha256"),
+                donation=text
+                # checksum=generate_password_hash(text, method="pbkdf2:sha256"),
             )
             # add to db
             db.session.add(new_submission)
             # make commit to db
             db.session.commit()
             flash("Text input received", category="success")
+            # results = db.session.query(RawData).filter_by(
+            #     donation='text').all()
+            # for result in results:
+            #     print(f"ID: {result.donor_id}, Donation: {result.donation}")
             # redirect to homepage
             return redirect(url_for("views.home"))
 

src/app/website/models.py

@@ -1,18 +1,50 @@
 from sqlalchemy.orm import Mapped, mapped_column, relationship
-from sqlalchemy import Integer, String, DateTime, ForeignKey  #, BLOB
-
-from sqlalchemy.sql import func
+from sqlalchemy import Integer, String, DateTime, ForeignKey, LargeBinary, type_coerce, Unicode, BLOB
+from sqlalchemy_utils import StringEncryptedType
+from sqlalchemy_utils.types.encrypted.encrypted_type import AesGcmEngine, AesEngine
+from sqlalchemy.dialects.mysql import VARBINARY, CHAR
+from sqlalchemy.types import TypeDecorator
+from sqlalchemy.sql import func, cast
 import datetime
+import cryptography
 from typing import List
 from . import db
 
+secret_key = "1234"
+
+class EncType(TypeDecorator):
+    impl = LargeBinary
+
+    def bind_expression(self, bindvalue):
+        return func.aes_encrypt(
+            type_coerce(bindvalue, CHAR()), func.unhex(func.sha2(secret_key, 512)),
+        )
+
+    def column_expression(self, col):
+        return cast(
+            func.aes_decrypt(col, func.unhex(func.sha2(secret_key, 512)),),
+            CHAR(charset="utf8"),
+        )
+
+
 
 # the raw data model
 class RawData(db.Model):
     # the submission id
     donor_id: Mapped[int] = mapped_column(primary_key=True)
     # should this be the donated data as zip?
-    donation: Mapped[str] = mapped_column(String(500), nullable=True)
+    # use all the emails as string and encrypt
+    # but somewhere we need to store the blob of the zip file
+    # here we could encrypt the whole column as it is never touched again
+    # if emails contain large attachements, could this overflow the database?
+    # donation: Mapped[str] = mapped_column(StringEncryptedType(
+                                    #    VARBINARY(5000),
+                                    #    secret_key,
+                                    #    AesGcmEngine,
+                                    #    AesEngine,
+                                    #    'pkcs5',
+                                    #    length=5000), nullable=True)
+    donation: Mapped[str] = mapped_column(EncType, nullable=True)
     # the hash checksum of the donation zip file, for example SHA-256
     checksum: Mapped[str] = mapped_column(String(128), nullable=True)
     # Now the metadata
@@ -21,7 +53,10 @@ class RawData(db.Model):
         DateTime(timezone=True), server_default=func.now(), nullable=False
     )
     # the email of the donor
-    email: Mapped[str] = mapped_column(String(500), nullable=True)
+    # email goes into different model for newsletter
+    # email: Mapped[str] = mapped_column(String(500), nullable=True)
+    # donor consent form
+    consent: Mapped[bool] = mapped_column(Integer, nullable=True)
     # the age group of the donor in categories
     age: Mapped[int] = mapped_column(Integer, nullable=True)
     # the region of the donor in categories
@@ -47,7 +82,22 @@ class ProcessedData(db.Model):
     date: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True), default=func.now(), nullable=False
     )
+    # date the email was sent
+    date_sent: Mapped[datetime.datetime] = mapped_column(
+        DateTime(timezone=True), default=func.now(), nullable=False
+    )
+    # if attachments were included
+    attachments: Mapped[bool] = mapped_column(Integer, nullable=False)
+    # type of the attachements
+    attachment_type: Mapped[str] = mapped_column(String(50), nullable=False)
     # the language of the email
     language: Mapped[str] = mapped_column(String(50), nullable=False)
     # the original donation id, one to many relationship
     donation_id: Mapped[int] = mapped_column(ForeignKey("raw_data.donor_id"))
+
+class InformantList(db.Model):
+    # the submission id
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    # the informant email
+    # should this be encrypted?
+    informant_email: Mapped[str] = mapped_column(String(500), nullable=False)