Add 'Federal Agency' to list of accepted states (#998)

## Fixes issue
#797

## Description of Changes
Added 'Federal Agency' to list of accepted states, corrected formatting
of numbers on browse screen, moved choice lists to `utils` folder, and
modified formatter definitions.

## Screenshots (if appropriate)
Browse:
<img width="662" alt="Screenshot 2023-08-01 at 4 07 34 PM"
src="https://github.com/lucyparsons/OpenOversight/assets/5885605/2318176e-3815-4040-b81a-a3e598ea861c">

Update:
<img width="593" alt="Screenshot 2023-08-01 at 4 09 19 PM"
src="https://github.com/lucyparsons/OpenOversight/assets/5885605/d2b569b1-e11e-4fdf-9160-672e3ed5a9fb">

View officer:
<img width="1159" alt="Screenshot 2023-08-01 at 4 09 33 PM"
src="https://github.com/lucyparsons/OpenOversight/assets/5885605/b0d821d7-9a88-4cd6-aa18-067e578d8111">

Header for listing officers:
<img width="791" alt="Screenshot 2023-08-01 at 4 09 54 PM"
src="https://github.com/lucyparsons/OpenOversight/assets/5885605/b430f38c-ab03-4d61-a543-eedc708213b4">


## Tests and linting
 - [x] This branch is up-to-date with the `develop` branch.
 - [x] `pytest` passes on my local development environment.
 - [x] `pre-commit` passes on my local development environment.

OpenOversight/app/filters.py

@@ -1,5 +1,5 @@
 """Contains all templates filters."""
-import datetime
+from datetime import datetime
 
 import bleach
 import markdown as _markdown
@@ -14,7 +14,7 @@
 def instantiate_filters(app: Flask):
     """Instantiate all template filters"""
 
-    def get_timezone():
+    def get_timezone() -> str:
         """Return the applicable timezone for the filter."""
         return (
             session[KEY_TIMEZONE]
@@ -23,15 +23,13 @@ def get_timezone():
         )
 
     @app.template_filter("capfirst")
-    def capfirst_filter(s):
+    def capfirst_filter(s: str) -> str:
         return s[0].capitalize() + s[1:]  # only change 1st letter
 
     @app.template_filter("get_age")
-    def get_age_from_birth_year(birth_year):
+    def get_age_from_birth_year(birth_year) -> int:
         if birth_year:
-            return int(
-                datetime.datetime.now(pytz.timezone(get_timezone())).year - birth_year
-            )
+            return int(datetime.now(pytz.timezone(get_timezone())).year - birth_year)
 
     @app.template_filter("field_in_query")
     def field_in_query(form_data, field):
@@ -42,24 +40,29 @@ class which will render the field accordion open.
         return " in " if form_data.get(field) else ""
 
     @app.template_filter("markdown")
-    def markdown(text):
+    def markdown(text: str) -> Markup:
         text = text.replace("\n", "  \n")  # make markdown not ignore new lines.
         html = bleach.clean(_markdown.markdown(text), markdown_tags, markdown_attrs)
         return Markup(html)
 
     @app.template_filter("local_date")
-    def local_date(value):
+    def local_date(value: datetime) -> str:
         """Convert UTC datetime.datetime into a localized date string."""
         return value.astimezone(pytz.timezone(get_timezone())).strftime("%b %d, %Y")
 
     @app.template_filter("local_date_time")
-    def local_date_time(value):
+    def local_date_time(value: datetime) -> str:
         """Convert UTC datetime.datetime into a localized date time string."""
         return value.astimezone(pytz.timezone(get_timezone())).strftime(
             "%I:%M %p on %b %d, %Y"
         )
 
     @app.template_filter("local_time")
-    def local_time(value):
+    def local_time(value: datetime) -> str:
         """Convert UTC datetime.datetime into a localized time string."""
         return value.astimezone(pytz.timezone(get_timezone())).strftime("%I:%M %p")
+
+    @app.template_filter("thousands_seperator")
+    def thousands_seperator(value: int) -> str:
+        """Convert int to string with the appropriately applied commas."""
+        return f"{value:,}"

OpenOversight/app/main/forms.py

@@ -30,15 +30,16 @@
 from wtforms_sqlalchemy.fields import QuerySelectField
 
 from OpenOversight.app.formfields import TimeField
-from OpenOversight.app.main.choices import (
+from OpenOversight.app.models.database import Officer
+from OpenOversight.app.utils.choices import (
     AGE_CHOICES,
+    DEPARTMENT_STATE_CHOICES,
     GENDER_CHOICES,
     LINK_CHOICES,
     RACE_CHOICES,
     STATE_CHOICES,
     SUFFIX_CHOICES,
 )
-from OpenOversight.app.models.database import Officer
 from OpenOversight.app.utils.db import dept_choices, unit_choices
 from OpenOversight.app.widgets import BootstrapListWidget, FormFieldWidget
 
@@ -199,9 +200,9 @@ class DepartmentForm(Form):
     )
     state = SelectField(
         "The law enforcement agency's home state",
-        choices=STATE_CHOICES,
+        choices=[("", "Please Select a State")] + DEPARTMENT_STATE_CHOICES,
         default="",
-        validators=[AnyOf(allowed_values(STATE_CHOICES))],
+        validators=[AnyOf(allowed_values(DEPARTMENT_STATE_CHOICES))],
     )
     jobs = FieldList(
         StringField("Job", default="", validators=[Regexp(r"\w*")]), label="Ranks"

OpenOversight/app/main/views.py

@@ -25,7 +25,6 @@
 from OpenOversight.app import limiter, sitemap
 from OpenOversight.app.auth.forms import LoginForm
 from OpenOversight.app.main import main
-from OpenOversight.app.main.choices import AGE_CHOICES, GENDER_CHOICES, RACE_CHOICES
 from OpenOversight.app.main.downloads import (
     assignment_record_maker,
     descriptions_record_maker,
@@ -73,6 +72,7 @@
     db,
 )
 from OpenOversight.app.utils.auth import ac_or_admin_required, admin_required
+from OpenOversight.app.utils.choices import AGE_CHOICES, GENDER_CHOICES, RACE_CHOICES
 from OpenOversight.app.utils.cloud import crop_image, upload_image_to_s3_and_store_in_db
 from OpenOversight.app.utils.constants import (
     ENCODING_UTF_8,
@@ -520,10 +520,6 @@ def classify_submission(image_id, contains_cops):
 def add_department():
     form = DepartmentForm()
     if form.validate_on_submit():
-        if not form.state.data:
-            flash(f"You must select a valid state for {form.name.data}.")
-            return redirect(url_for("main.add_department"))
-
         department_does_not_exist = (
             Department.query.filter_by(
                 name=form.name.data, state=form.state.data
@@ -583,12 +579,6 @@ def edit_department(department_id):
     form = EditDepartmentForm(obj=department)
     original_ranks = department.jobs
     if form.validate_on_submit():
-        if not form.state.data:
-            flash(f"You must select a valid state for {form.name.data}.")
-            return redirect(
-                url_for("main.edit_department", department_id=department_id)
-            )
-
         if form.name.data != previous_name:
             does_already_department_exist = (
                 Department.query.filter_by(

OpenOversight/app/models/database.py

@@ -13,6 +13,7 @@
 from sqlalchemy.sql import func as sql_func
 from werkzeug.security import check_password_hash, generate_password_hash
 
+from OpenOversight.app.utils.choices import GENDER_CHOICES, RACE_CHOICES
 from OpenOversight.app.utils.constants import (
     ENCODING_UTF_8,
     HOUR,
@@ -233,7 +234,6 @@ def full_name(self):
     def race_label(self):
         if self.race is None:
             return "Data Missing"
-        from OpenOversight.app.main.choices import RACE_CHOICES
 
         for race, label in RACE_CHOICES:
             if self.race == race:
@@ -242,7 +242,6 @@ def race_label(self):
     def gender_label(self):
         if self.gender is None:
             return "Data Missing"
-        from OpenOversight.app.main.choices import GENDER_CHOICES
 
         for gender, label in GENDER_CHOICES:
             if self.gender == gender:

OpenOversight/app/models/database_imports.py

@@ -3,7 +3,6 @@
 import dateutil.parser
 
 from OpenOversight.app import login_manager
-from OpenOversight.app.main import choices
 from OpenOversight.app.models.database import (
     Assignment,
     Incident,
@@ -15,6 +14,12 @@
     User,
     db,
 )
+from OpenOversight.app.utils.choices import (
+    GENDER_CHOICES,
+    LINK_CHOICES,
+    RACE_CHOICES,
+    SUFFIX_CHOICES,
+)
 from OpenOversight.app.utils.general import get_or_create, str_is_true
 from OpenOversight.app.validators import state_validator, url_validator
 
@@ -76,9 +81,9 @@ def create_officer_from_dict(data: Dict[str, Any], force_id: bool = False) -> Of
         last_name=parse_str(data.get("last_name", "")),
         first_name=parse_str(data.get("first_name", "")),
         middle_initial=parse_str(data.get("middle_initial", "")),
-        suffix=validate_choice(data.get("suffix", ""), choices.SUFFIX_CHOICES),
-        race=validate_choice(data.get("race"), choices.RACE_CHOICES),
-        gender=validate_choice(data.get("gender"), choices.GENDER_CHOICES),
+        suffix=validate_choice(data.get("suffix", ""), SUFFIX_CHOICES),
+        race=validate_choice(data.get("race"), RACE_CHOICES),
+        gender=validate_choice(data.get("gender"), GENDER_CHOICES),
         employment_date=parse_date(data.get("employment_date")),
         birth_year=parse_int(data.get("birth_year")),
         unique_internal_identifier=parse_str(
@@ -103,11 +108,11 @@ def update_officer_from_dict(data: Dict[str, Any], officer: Officer) -> Officer:
     if "middle_initial" in data.keys():
         officer.middle_initial = parse_str(data.get("middle_initial", ""))
     if "suffix" in data.keys():
-        officer.suffix = validate_choice(data.get("suffix", ""), choices.SUFFIX_CHOICES)
+        officer.suffix = validate_choice(data.get("suffix", ""), SUFFIX_CHOICES)
     if "race" in data.keys():
-        officer.race = validate_choice(data.get("race"), choices.RACE_CHOICES)
+        officer.race = validate_choice(data.get("race"), RACE_CHOICES)
     if "gender" in data.keys():
-        officer.gender = validate_choice(data.get("gender"), choices.GENDER_CHOICES)
+        officer.gender = validate_choice(data.get("gender"), GENDER_CHOICES)
     if "employment_date" in data.keys():
         officer.employment_date = parse_date(data.get("employment_date"))
     if "birth_year" in data.keys():
@@ -193,7 +198,7 @@ def create_link_from_dict(data: Dict[str, Any], force_id: bool = False) -> Link:
     link = Link(
         title=data.get("title", ""),
         url=url_validator(data["url"]),
-        link_type=validate_choice(data.get("link_type"), choices.LINK_CHOICES),
+        link_type=validate_choice(data.get("link_type"), LINK_CHOICES),
         description=parse_str(data.get("description"), None),
         author=parse_str(data.get("author"), None),
         creator_id=parse_int(data.get("creator_id")),
@@ -216,7 +221,7 @@ def update_link_from_dict(data: Dict[str, Any], link: Link) -> Link:
     if "url" in data:
         link.url = url_validator(data["url"])
     if "link_type" in data:
-        link.link_type = validate_choice(data.get("link_type"), choices.LINK_CHOICES)
+        link.link_type = validate_choice(data.get("link_type"), LINK_CHOICES)
     if "description" in data:
         link.description = parse_str(data.get("description"), None)
     if "author" in data:

OpenOversight/app/templates/browse.html

@@ -27,11 +27,11 @@ <h2>
               {% endif %}
             </h2>
             <div>
-              <b class="dept-{{ department.id }}">Officers Documented:</b> {{ department.total_documented_officers() }}
+              <b class="dept-{{ department.id }}">Officers Documented:</b> {{ department.total_documented_officers() | thousands_seperator }}
               <br>
-              <b class="dept-{{ department.id }}">Assignments Documented:</b> {{ department.total_documented_assignments() }}
+              <b class="dept-{{ department.id }}">Assignments Documented:</b> {{ department.total_documented_assignments() | thousands_seperator }}
               <br>
-              <b class="dept-{{ department.id }}">Incidents Documented:</b> {{ department.total_documented_incidents() }}
+              <b class="dept-{{ department.id }}">Incidents Documented:</b> {{ department.total_documented_incidents() | thousands_seperator }}
             </div>
             <p>
               <a class="btn btn-lg btn-primary"

OpenOversight/app/templates/list_officer.html

@@ -13,7 +13,11 @@
 {% endblock head %}
 {% block content %}
   <div class="container" role="main">
-    <h1>{{ department.name | title }} Officers</h1>
+    {% if department.state %}
+      <h1>[{{ department.state }}] {{ department.name | title }} Officers</h1>
+    {% else %}
+      <h1>{{ department.name | title }} Officers</h1>
+    {% endif %}
     <div class="row">
       <div class="filter-sidebar col-sm-3">
         <h3 class="sidebar-title">Filter officers</h3>

OpenOversight/app/templates/partials/officer_general_information.html

@@ -34,7 +34,11 @@ <h3>
           <td>
             <b>Department</b>
           </td>
-          <td>{{ officer.department.name }}</td>
+          {% if officer.department.state %}
+            <td>[{{ officer.department.state }}] {{ officer.department.name }}</td>
+          {% else %}
+            <td>{{ officer.department.name }}</td>
+          {% endif %}
         </tr>
         <tr>
           <td>

OpenOversight/app/main/choices.py → OpenOversight/app/utils/choices.py

@@ -1,26 +1,8 @@
+"""Contains choice lists of (value, label) tuples for form Select fields."""
 from us import states
 
 
-# Choices are a list of (value, label) tuples
-SUFFIX_CHOICES = [
-    ("", "-"),
-    ("Jr", "Jr"),
-    ("Sr", "Sr"),
-    ("II", "II"),
-    ("III", "III"),
-    ("IV", "IV"),
-    ("V", "V"),
-]
-RACE_CHOICES = [
-    ("BLACK", "Black"),
-    ("WHITE", "White"),
-    ("ASIAN", "Asian"),
-    ("HISPANIC", "Hispanic"),
-    ("NATIVE AMERICAN", "Native American"),
-    ("PACIFIC ISLANDER", "Pacific Islander"),
-    ("Other", "Other"),
-    ("Not Sure", "Not Sure"),
-]
+AGE_CHOICES = [(str(age), str(age)) for age in range(16, 101)]
 
 GENDER_CHOICES = [
     ("Not Sure", "Not Sure"),
@@ -29,13 +11,33 @@
     ("Other", "Other"),
 ]
 
-STATE_CHOICES = [("", "Please Select a State")] + [
-    (state.abbr, state.name) for state in states.STATES
-]
 LINK_CHOICES = [
     ("", ""),
     ("link", "Link"),
     ("video", "YouTube Video"),
     ("other_video", "Other Video"),
 ]
-AGE_CHOICES = [(str(age), str(age)) for age in range(16, 101)]
+
+RACE_CHOICES = [
+    ("BLACK", "Black"),
+    ("WHITE", "White"),
+    ("ASIAN", "Asian"),
+    ("HISPANIC", "Hispanic"),
+    ("NATIVE AMERICAN", "Native American"),
+    ("PACIFIC ISLANDER", "Pacific Islander"),
+    ("Other", "Other"),
+    ("Not Sure", "Not Sure"),
+]
+
+STATE_CHOICES = [(state.abbr, state.name) for state in states.STATES]
+DEPARTMENT_STATE_CHOICES = [("FA", "Federal Agency")] + STATE_CHOICES
+
+SUFFIX_CHOICES = [
+    ("", "-"),
+    ("Jr", "Jr"),
+    ("Sr", "Sr"),
+    ("II", "II"),
+    ("III", "III"),
+    ("IV", "IV"),
+    ("V", "V"),
+]

OpenOversight/app/utils/forms.py

@@ -4,7 +4,6 @@
 from sqlalchemy.orm import selectinload
 from sqlalchemy.sql.expression import cast
 
-from OpenOversight.app.main.choices import GENDER_CHOICES, RACE_CHOICES
 from OpenOversight.app.models.database import (
     Assignment,
     Description,
@@ -20,6 +19,7 @@
     Unit,
     db,
 )
+from OpenOversight.app.utils.choices import GENDER_CHOICES, RACE_CHOICES
 from OpenOversight.app.utils.general import get_or_create
 
 

OpenOversight/tests/conftest.py

@@ -3,6 +3,7 @@
 import math
 import os
 import random
+import string
 import sys
 import threading
 import time
@@ -38,19 +39,34 @@
     User,
 )
 from OpenOversight.app.models.database import db as _db
+from OpenOversight.app.utils.choices import DEPARTMENT_STATE_CHOICES
 from OpenOversight.app.utils.constants import ENCODING_UTF_8
 from OpenOversight.app.utils.general import merge_dicts
 from OpenOversight.tests.routes.route_helpers import ADMIN_EMAIL, ADMIN_PASSWORD
-from OpenOversight.tests.test_utils import PoliceDepartment
 
 
 factory = Faker()
 
+
+class PoliceDepartment:
+    """Base Police Department class."""
+
+    def __init__(self, name, short_name, state="", unique_internal_identifier_label=""):
+        self.name = name
+        self.short_name = short_name
+        self.state = state if state else random.choice(DEPARTMENT_STATE_CHOICES)[0]
+        self.unique_internal_identifier_label = (
+            unique_internal_identifier_label
+            if unique_internal_identifier_label
+            else "".join(random.choices(string.ascii_uppercase + string.digits, k=20))
+        )
+
+
 OFFICERS = [
     ("IVANA", "", "TINKLE"),
     ("SEYMOUR", "", "BUTZ"),
     ("HAYWOOD", "U", "CUDDLEME"),
-    ("BEA", "", "O'PROBLEM"),
+    ("BEA", "", "PROBLEM"),
     ("URA", "", "SNOTBALL"),
     ("HUGH", "", "JASS"),
 ]