-
Notifications
You must be signed in to change notification settings - Fork 685
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add journalist interface API #3619
Changes from all commits
4e90665
8068543
b10dbe3
55241d9
0a7fcbb
3402155
efad309
98d43d2
efe2726
dac442c
79e5b60
f6a42c5
35d032e
d32da8d
73e0aa7
122664e
8b425db
8ae05b1
3504688
1324687
b4e9bb2
d2c1ce3
1fef4da
15baf24
4fc88f7
244ff82
4f93e8f
6d069a8
7adb254
4596452
d6765c8
024ff35
1d43208
183cc3b
eea5e9e
b2729f8
2621cf4
3be0707
586769a
2002347
978f584
8cf6024
3ab39c7
3908748
545c9d1
02432ed
b490778
172e0fb
b6ad7cd
e11cf0f
f34f08c
0d86a30
0b9f6c6
785d4d2
8685884
52d6c5a
c324abe
daecdd7
b6d239e
8f385a9
664be6e
ebe1553
b5447d7
25cec03
900f3b3
7e7a447
5ab60f7
dc21b3c
1b852b0
57dd181
e42c7f1
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
"""Create source UUID column | ||
|
||
Revision ID: 3d91d6948753 | ||
Revises: faac8092c123 | ||
Create Date: 2018-07-09 22:39:05.088008 | ||
|
||
""" | ||
from alembic import op | ||
import sqlalchemy as sa | ||
from sqlalchemy.sql import quoted_name | ||
import subprocess | ||
import uuid | ||
|
||
# revision identifiers, used by Alembic. | ||
revision = '3d91d6948753' | ||
down_revision = 'faac8092c123' | ||
branch_labels = None | ||
depends_on = None | ||
|
||
|
||
def upgrade(): | ||
# Schema migration | ||
op.rename_table('sources', 'sources_tmp') | ||
|
||
# Add UUID column. | ||
op.add_column('sources_tmp', sa.Column('uuid', sa.String(length=36))) | ||
|
||
# Add UUIDs to sources_tmp table. | ||
conn = op.get_bind() | ||
sources = conn.execute(sa.text("SELECT * FROM sources_tmp")).fetchall() | ||
|
||
for source in sources: | ||
id = source.id | ||
source_uuid = str(uuid.uuid4()) | ||
conn.execute( | ||
sa.text("UPDATE sources_tmp SET uuid=('{}') WHERE id={}".format( | ||
source_uuid, id))) | ||
|
||
# Now create new table with unique constraint applied. | ||
op.create_table(quoted_name('sources', quote=False), | ||
sa.Column('id', sa.Integer(), nullable=False), | ||
sa.Column('uuid', sa.String(length=36), nullable=False), | ||
sa.Column('filesystem_id', sa.String(length=96), nullable=True), | ||
sa.Column('journalist_designation', sa.String(length=255), | ||
nullable=False), | ||
sa.Column('flagged', sa.Boolean(), nullable=True), | ||
sa.Column('last_updated', sa.DateTime(), nullable=True), | ||
sa.Column('pending', sa.Boolean(), nullable=True), | ||
sa.Column('interaction_count', sa.Integer(), nullable=False), | ||
sa.PrimaryKeyConstraint('id'), | ||
sa.UniqueConstraint('uuid'), | ||
sa.UniqueConstraint('filesystem_id') | ||
) | ||
|
||
# Data Migration: move all sources into the new table. | ||
conn.execute(''' | ||
INSERT INTO sources | ||
SELECT id, uuid, filesystem_id, journalist_designation, flagged, | ||
last_updated, pending, interaction_count | ||
FROM sources_tmp | ||
''') | ||
|
||
# Now delete the old table. | ||
op.drop_table('sources_tmp') | ||
|
||
|
||
def downgrade(): | ||
with op.batch_alter_table('sources', schema=None) as batch_op: | ||
batch_op.drop_column('uuid') |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
"""create submission uuid column | ||
|
||
Revision ID: fccf57ceef02 | ||
Revises: 3d91d6948753 | ||
Create Date: 2018-07-12 00:06:20.891213 | ||
|
||
""" | ||
from alembic import op | ||
import sqlalchemy as sa | ||
|
||
import uuid | ||
|
||
# revision identifiers, used by Alembic. | ||
revision = 'fccf57ceef02' | ||
down_revision = '3d91d6948753' | ||
branch_labels = None | ||
depends_on = None | ||
|
||
|
||
def upgrade(): | ||
# Schema migration | ||
op.rename_table('submissions', 'submissions_tmp') | ||
|
||
# Add UUID column. | ||
op.add_column('submissions_tmp', sa.Column('uuid', sa.String(length=36))) | ||
|
||
# Add UUIDs to submissions_tmp table. | ||
conn = op.get_bind() | ||
submissions = conn.execute( | ||
sa.text("SELECT * FROM submissions_tmp")).fetchall() | ||
|
||
for submission in submissions: | ||
id = submission.id | ||
submission_uuid = str(uuid.uuid4()) | ||
conn.execute( | ||
sa.text("""UPDATE submissions_tmp | ||
SET uuid=('{}') | ||
WHERE id={}""".format(submission_uuid, id))) | ||
|
||
# Now create new table with unique constraint applied. | ||
op.create_table('submissions', | ||
sa.Column('id', sa.Integer(), nullable=False), | ||
sa.Column('uuid', sa.String(length=36), nullable=False), | ||
sa.Column('source_id', sa.Integer(), nullable=True), | ||
sa.Column('filename', sa.String(length=255), nullable=False), | ||
sa.Column('size', sa.Integer(), nullable=False), | ||
sa.Column('downloaded', sa.Boolean(), nullable=True), | ||
sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ), | ||
sa.PrimaryKeyConstraint('id'), | ||
sa.UniqueConstraint('uuid') | ||
) | ||
|
||
# Data Migration: move all submissions into the new table. | ||
conn.execute(''' | ||
INSERT INTO submissions | ||
SELECT id, uuid, source_id, filename, size, downloaded | ||
FROM submissions_tmp | ||
''') | ||
|
||
# Now delete the old table. | ||
op.drop_table('submissions_tmp') | ||
|
||
|
||
def downgrade(): | ||
with op.batch_alter_table('submissions', schema=None) as batch_op: | ||
batch_op.drop_column('uuid') |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,19 +1,21 @@ | ||
# -*- coding: utf-8 -*- | ||
|
||
from datetime import datetime, timedelta | ||
from flask import Flask, session, redirect, url_for, flash, g, request | ||
from flask import (Flask, session, redirect, url_for, flash, g, request, | ||
render_template) | ||
from flask_assets import Environment | ||
from flask_babel import gettext | ||
from flask_wtf.csrf import CSRFProtect, CSRFError | ||
from os import path | ||
from werkzeug.exceptions import default_exceptions # type: ignore | ||
|
||
import i18n | ||
import template_filters | ||
import version | ||
|
||
from crypto_util import CryptoUtil | ||
from db import db | ||
from journalist_app import account, admin, main, col | ||
from journalist_app import account, admin, api, main, col | ||
from journalist_app.utils import get_source, logged_in | ||
from models import Journalist | ||
from store import Storage | ||
|
@@ -39,7 +41,7 @@ def create_app(config): | |
app.config.from_object(config.JournalistInterfaceFlaskConfig) | ||
app.sdconfig = config | ||
|
||
CSRFProtect(app) | ||
csrf = CSRFProtect(app) | ||
Environment(app) | ||
|
||
if config.DATABASE_ENGINE == "sqlite": | ||
|
@@ -80,6 +82,18 @@ def handle_csrf_error(e): | |
flash(msg, 'error') | ||
return redirect(url_for('main.login')) | ||
|
||
def _handle_http_exception(error): | ||
# Workaround for no blueprint-level 404/5 error handlers, see: | ||
# https://github.com/pallets/flask/issues/503#issuecomment-71383286 | ||
handler = app.error_handler_spec['api'][error.code].values()[0] | ||
if request.path.startswith('/api/') and handler: | ||
return handler(error) | ||
|
||
return render_template('error.html', error=error), error.code | ||
|
||
for code in default_exceptions: | ||
app.errorhandler(code)(_handle_http_exception) | ||
|
||
i18n.setup_app(config, app) | ||
|
||
app.jinja_env.trim_blocks = True | ||
|
@@ -97,17 +111,6 @@ def handle_csrf_error(e): | |
template_filters.rel_datetime_format | ||
app.jinja_env.filters['filesizeformat'] = template_filters.filesizeformat | ||
|
||
@app.template_filter('autoversion') | ||
def autoversion_filter(filename): | ||
"""Use this template filter for cache busting""" | ||
absolute_filename = path.join(config.SECUREDROP_ROOT, filename[1:]) | ||
if path.exists(absolute_filename): | ||
timestamp = str(path.getmtime(absolute_filename)) | ||
else: | ||
return filename | ||
versioned_filename = "{0}?v={1}".format(filename, timestamp) | ||
return versioned_filename | ||
|
||
@app.before_request | ||
def setup_g(): | ||
"""Store commonly used values in Flask's special g object""" | ||
|
@@ -130,8 +133,11 @@ def setup_g(): | |
g.html_lang = i18n.locale_to_rfc_5646(g.locale) | ||
g.locales = i18n.get_locale2name() | ||
|
||
if request.endpoint not in _insecure_views and not logged_in(): | ||
return redirect(url_for('main.login')) | ||
if request.path.split('/')[1] == 'api': | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: I think doing |
||
pass # We use the @token_required decorator for the API endpoints | ||
else: # We are not using the API | ||
if request.endpoint not in _insecure_views and not logged_in(): | ||
return redirect(url_for('main.login')) | ||
|
||
if request.method == 'POST': | ||
filesystem_id = request.form.get('filesystem_id') | ||
|
@@ -144,5 +150,8 @@ def setup_g(): | |
url_prefix='/account') | ||
app.register_blueprint(admin.make_blueprint(config), url_prefix='/admin') | ||
app.register_blueprint(col.make_blueprint(config), url_prefix='/col') | ||
api_blueprint = api.make_blueprint(config) | ||
app.register_blueprint(api_blueprint, url_prefix='/api/v1') | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I actually don't think we should attach the API to the main app. They are different enough that I think it would be reasonable to have two applications and use Apache/Nginx to correctly proxy between the two. That said, we can't really do that now because we don't control the configs. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. One of the nice reasons to implement the API as a blueprint registered on the main journalist app is that it's extremely simple to deploy and does not add complexity to an already quite complex ops story. Is there a major advantage to keeping the two separate that would justify the Apache config changes? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. From an ops perspective (in the context of SD), I think this is the correct thing to do. I just generally like to keep the apps separate when I deploy things because it prevents surprise behavior between different components IMO. Probably should have clarified my initial comment to better say "I don't like this particularly but also what you did is the only sane thing to do." |
||
csrf.exempt(api_blueprint) | ||
|
||
return app |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
👍