diff --git a/docs/advanced_csv_import.md b/docs/advanced_csv_import.md new file mode 100644 index 000000000..589ebfac3 --- /dev/null +++ b/docs/advanced_csv_import.md @@ -0,0 +1,206 @@ +Advanced csv import +============================================= +This is the documentation of the advanced csv import command `advanced_csv_import`. For the documentation of the less complicated +bulk-upload command `bulk-add-officers` see [`bulk_upload`](bulk_upload.md). `bulk-add-officers` accepts one csv containing information +about the officer, including badge-number, jobs and salary and makes decisions on whether to update rows in the database +or create new entries based on the existing data. + +The advanced csv upload is for the most part a way to copy data for one department into the database with as little as possible logic added on. +So the tables provided in csv form represent the data that will be inside the sql tables after running the command. +(With a few exceptions for many-to-many relationships and auxiliary models like location and license plates) + +Before you start +---------------- +CSV uploads should always be tested locally or in other non-production environments, and it is strongly recommended +to have the database backed up before running the command. The command is designed to fail early and will +only commit the changes if it didn't encounter any problems. The command however is pretty powerful +and can therefore lead to data loss and inconsistencies if the provided csv files are not prepared correctly. + +Explanation of the command +-------------------------- +```shell + /usr/src/app/OpenOversight$ flask advanced-csv-import --help + Usage: flask advanced-csv-import [OPTIONS] DEPARTMENT_NAME DEPARTMENT_STATE + + Add or update officers, assignments, salaries, links and incidents from + csv files in the department using the DEPARTMENT_NAME and DEPARTMENT_STATE. + + The csv files are treated as the source of truth. Existing entries might + be overwritten as a result, backing up the database and running the + command locally first is highly recommended. + + See the documentation before running the command. + + Options: + --officers-csv PATH + --assignments-csv PATH + --salaries-csv PATH + --links-csv PATH + --incidents-csv PATH + --force-create Only for development/testing! + --overwrite-assignments + --help Show this message and exit. +``` + + +The command expects two mandatory arguments, the department name and department state. +This is to reduce the chance of making changes to the wrong department by mixing up files. +Then there are 5 options to include paths to officers, assignments, salaries, incidents and links csv files. +Then there is a `--force-create` flag that allows to delete and overwrite existing entries in the database. +This is only supposed to be used in non-production environments and to allow replication of the data of another (in most cases production) +instance to local environments to test the import command locally first. More details on that flag at the end of the document: :ref:`ref-aci-force-create`. +Finally, there is `--overwrite-assignments` which simplifies updating assignments. Instead of updating them, +all assignments for the relevant officers are deleted and created new based on the provided data. This flag is only +considered if an assignments-csv is provided and ignored otherwise. See the instructions in +the section on assignment-csv for more details. + +General overview of the csv import +----------------------------------- +The following lists the header fields that each csv can contain. If the csv includes any other fields, the command will fail. +However, the fields are not case-sensitive and spaces are treated as `_`. So `Officer ID` can be used instead of `officer_id`. + +*All optional fields can be left blank and will be inserted as* `NULL` *or empty string as appropriate.* +**Warning:** When updating a record a field that is left blank might overwrite an existing record. +This can only be prevented by not including the column in the csv at all. + +.. _ref-aci-formats: + +Formats: +- `date` - The date should be provided in `YYYY-MM-DD` format. +- `time` - Time should be provided in `HH:MM:SS` 24h-format in the respective timezone. +- `DEPARTMENT_STATE` - The department state should be provided in the `standard two-letter abbreviation `_ format. + + +The `id` field +-------------- +Each csv corresponds to a table in the OpenOversight database. And each csv file has to include `id` as a field in the table. +That field has one primary purpose: If the field is blank, it is assumed that that row is a new entry. +If the field contains a number however, it is assumed that a record with that particular id already exists in the database +and the record will be updated according to the provided fields. Finally, in the case of officers and incidents +there is a third option where the `id` field can contain a string that starts with `#`. This also indicates a new record, +but that new record can be referenced in other provided tables. (for example as the `officer_id` in the salaries csv) + + + +Officers csv +------------ +- Required: `id, department_name, department_state` +- Optional: `last_name, first_name, middle_initial, suffix, race, gender, employment_date, birth_year, unique_internal_identifier` +- Ignored: `badge_number, job_title, most_recent_salary, unique_identifier` (Unused but command will not fail when field is present) + +Details +-------- +- `department_name` - Name of department exactly as it is in the server database. + This needs to match the department name provided with the command. +- `department_state` - Name of department state exactly as it is in the server database, which will be the + [standard two-letter abbreviation](https://www.faa.gov/air_traffic/publications/atpubs/cnt_html/appendix_a.html) for the department's respective location. + This needs to match the department state provided with the command. +- `unique_internal_identifier` - A string or number that can be used to + uniquely identify the officer, in departments in which the badge + number stays with the officer using that number is fine. Can and should be left blank + if no such number is available. +- `first_name` & `last_name` Will be inserted into the database as is. +- `middle_initial` - Usually up to one character, but can be more. +- `suffix` - Choice of the following values: `Jr, Sr, II, III, IV, V`. +- `gender` - One of the following values: `M`, `F`, `Other`. +- `race` - One of the following values: `BLACK`, `WHITE`, `ASIAN`, `HISPANIC`, `NATIVE AMERICAN`, `PACIFIC ISLANDER`, `Other`. +- `employment_date` - [Date](https://help.highbond.com/helpdocs/analytics/13/user-guide/en-us/Content/table_definition/c_formats_of_date_and_time_source_data.htm) representing the start of employment with this department. +- `birth_year` - Integer representing the birth year of the officer in a `yyyy` format. + +Assignments csv +--------------- +- Required: `id, officer_id, job_title` +- Optional: `badge_number, unit_id, unit_name, start_date, resign_date` + +Details +-------- +- `officer_id` - Number referring to `id` of existing officer or string starting with `#` referring to a newly created officer in the provided officers csv. +- `badge_number` - Any string that represents the star or badge number of the officer. In some departments this number changes with the assignment. +- `job_title` - The job title, will be created if it does not exist. +- `unit_id` - ID of existing unit within the department. +- `unit_name` - Name of the unit, only used if the `unit_id` column is not provided. +- `start_date` - [Start date](https://help.highbond.com/helpdocs/analytics/13/user-guide/en-us/Content/table_definition/c_formats_of_date_and_time_source_data.htm) of this assignment. +- `resign_date` - [End date](https://help.highbond.com/helpdocs/analytics/13/user-guide/en-us/Content/table_definition/c_formats_of_date_and_time_source_data.htm) of this assignment. + +Special Flag +------------- +The `--overwrite-assignments` in the command can be used to not merge new with existing assignments. +Instead, all existing assignments belonging to officers named in the `officer_id` column are deleted first, +before the new assignments contained in the provided csv are created in the database. + +This should only be used if the provided csv contains both the currently in the database and additional assignments, +or is based on a better and more complete dataset, for example after receiving a dataset for historic assignment data. + +Salaries csv +------------ +- Required: `id, officer_id, salary, year` +- Optional: `overtime_pay, is_fiscal_year` + +Details +-------- +- `officer_id` - Integer referring to `id` of existing officer or string starting with `#` referring to a newly created officer in the provided officers csv. +- `salary` - Number representing the officer's salary in the given year. +- `year` - Integer, the year this salary information refers to. +- `overtime_pay` - Number representing the amount of overtime payment for offer in given year. +- `is_fiscal_year` - Boolean value, indicating whether the provided year refers to calendar year or fiscal year. + The values `true`, `t`, `yes` and `y` are treated as "yes, the salary is for the fiscal year", all others (including blank) as "no". + +Incidents csv +------------- +- Required: `id, department_name, department_state` +- Optional: `date, time, report_number, description, street_name, cross_street1, cross_street2, city, state, zip_code, + created_by, last_updated_by, officer_ids, license_plates` + +Details +-------- +- `department_name` - Name of department exactly as in the server database. + This needs to match the department name provided with the command. +- `department_state` - Name of department state exactly as it is in the server database, which will be the + `standard two-letter abbreviation `_ for the department's respective location. +- `date` - [Date](https://help.highbond.com/helpdocs/analytics/13/user-guide/en-us/Content/table_definition/c_formats_of_date_and_time_source_data.htm) of the incident +- `time` - [Time](https://help.highbond.com/helpdocs/analytics/13/user-guide/en-us/Content/table_definition/c_formats_of_date_and_time_source_data.htm) of the incident +- `report_number` - String representing any kind of number assigned to complaints or incidents by the police department. +- `description` - Text description of the incident. +- `street_name` - Name of the street the incident occurred, but should not include the street number. +- `cross_street1`, `cross_street2` The two closest intersecting streets. +- `city`, `state`, `zip_code` State needs to be in 2 letter abbreviated notation. +- `created_by`, `last_updated_by` - ID of existing user shown as responsible for adding this entry. +- `officer_ids` - IDs of officers involved in the incident, separated by `|`. + - Each individual id can either be an integer referring to an existing officer or a string starting with `#` referring to a newly created officer. + - Example: `123|#C1|1627` for three officers, one with id 123, one with 1627 and one whose record was created in the officers csv + and whose id-field was the string `#C1`. + +- `license_plates` - All license plates involved in the incident. If there is more than one, they can be separated with a `|`. + - Each license plate consists of the license plate number and optionally a state in abbreviated form separated by an underscore `_`. + - Example: `ABC123_IL|B991` for one license plate with number `ABC123` from Illinois and one with number `B991` and no associated state. + + +Links csv +--------- +- Required: `id, url` +- Optional: `title, link_type, description, author, created_by, officer_ids, incident_ids` + +Details +-------- +- `url` - Full url of the link starting with `http://` or `https://`. +- `title` - Text that will be displayed as the link. +- `description` - A short description of the link. +- `link_type` - Choice of `Link`, `YouTube Video` and `Other Video`. +- `author` - The source or author of the linked article, report, video. +- `created_by` - ID of existing user shown as responsible for adding this entry. +- `officer_ids` - IDs of officer profiles this link should be visible on, separated by `|`. See same field in incidents above for more details. +- `incidents_ids` - IDs of incidents this link should be associated with, separated by `|`. Just like `officer_ids` this can contain strings. + starting with `#` to refer to an incident created in the incident csv. + +Examples +--------- +Example csvs can be found in the repository under `OpenOversight/tests/test_csvs`. + +Local development flag `--force-create` +--------------------------------------- +This flag changes the behavior when an integer is provided as `id`. Instead of updating an existing record, +a new record will be created and assigned the given `id`. If a record with that `id` already exists in the +database, it will be deleted before the new record is created. + +This functionality is intended to be used to import csv files downloaded from `OpenOversight download page `_ +to get a local copy of the production data for one department in the local development database. diff --git a/docs/advanced_csv_import.rst b/docs/advanced_csv_import.rst deleted file mode 100644 index 52840ed9a..000000000 --- a/docs/advanced_csv_import.rst +++ /dev/null @@ -1,211 +0,0 @@ -Advanced csv import -============================================= -This is the documentation of the advanced csv import command ``advanced_csv_import``. For the documentation of the less complicated -bulk-upload command ``bulk-add-officers`` see :doc:`bulk_upload`. ``bulk-add-officers`` accepts one csv containing information -about the officer, including badge-number, jobs and salary and makes decisions on whether to update rows in the database -or create new entries based on the existing data. - -The advanced csv upload is for the most part a way to copy data for one department into the database with as little as possible logic added on. -So the tables provided in csv form represent the data that will be inside the sql tables after running the command. -(With a few exceptions for many-to-many relationships and auxiliary models like location and license plates) - -Before you start ----------------- -CSV uploads should always be tested locally or in other non-production environments and it is strongly recommended -to have the database backed up before running the command. The command is designed to fail early and will -only commit the changes if it didn't encounter any problems. The command however is pretty powerful -and can therefore lead to data loss and inconsistencies if the provided csv files are not prepared correctly. - -Explanation of the command --------------------------- -:: - - /usr/src/app/OpenOversight$ flask advanced-csv-import --help - Usage: flask advanced-csv-import [OPTIONS] DEPARTMENT_NAME DEPARTMENT_STATE - - Add or update officers, assignments, salaries, links and incidents from - csv files in the department using the DEPARTMENT_NAME and DEPARTMENT_STATE. - - The csv files are treated as the source of truth. Existing entries might - be overwritten as a result, backing up the database and running the - command locally first is highly recommended. - - See the documentation before running the command. - - Options: - --officers-csv PATH - --assignments-csv PATH - --salaries-csv PATH - --links-csv PATH - --incidents-csv PATH - --force-create Only for development/testing! - --overwrite-assignments - --help Show this message and exit. - - -The command expects two mandatory arguments, the department name and department state. -This is to reduce the chance of making changes to the wrong department by mixing up files. -Then there are 5 options to include paths to officers, assignments, salaries, incidents and links csv files. -Then there is a ``--force-create`` flag that allows to delete and overwrite existing entries in the database. -This is only supposed to be used in non-production environments and to allow replication of the data of another (in most cases production) -instance to local environments to test the import command locally first. More details on that flag at the end of the document: :ref:`ref-aci-force-create`. -Finally there is ``--overwrite-assignments`` which simplifies updating assignments. Instead of updating them, -all assignments for the relevant officers are deleted and created new based on the provided data. This flag is only -considered if an assignments-csv is provided and ignored otherwise. See the instructions in -the section on assignment-csv for more details. - -General overview of the csv import: -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The following lists the header fields that each csv can contain. If the csv includes any other fields, the command will fail. -However the fields are not case-sensitive and spaces are treated as ``_``. So ``Officer ID`` can be used instead of ``officer_id``. - -*All optional fields can be left blank and will be inserted as* ``NULL`` *or empty string as appropriate.* -**Warning:** When updating a record a field that is left blank might overwrite an existing record. -This can only be prevented by not including the column in the csv at all. - -.. _ref-aci-formats: - -Formats -~~~~~~~~~~~~ -- date: The date should be provided in ``YYYY-MM-DD`` format. -- time: Time should be provided in ``HH:MM:SS`` 24h-format in the respective timezone. -- ``DEPARTMENT_STATE``: The department state should be provided in the `standard two-letter abbreviation `_ format. - - -The ``id`` field -~~~~~~~~~~~~~~~~~ -Each csv corresponds to a table in the OpenOversight database. And each csv file has to include ``id`` as a field in the table. -That field has one primary purpose: If the field is blank, it is assumed that that row is a new entry. -If the field contains a number however, it is assumed that a record with that particular id already exists in the database -and the record will be updated according to the provided fields. Finally in the case of officers and incidents -there is a third option where the ``id`` field can contain a string that starts with ``#``. This also indicates a new record, -but that new record can be referenced in other provided tables. (for example as the ``officer_id`` in the salaries csv) - - - -Officers csv -^^^^^^^^^^^^ -- Required: ``id, department_name, department_state`` -- Optional: ``last_name, first_name, middle_initial, suffix, race, gender, employment_date, birth_year, unique_internal_identifier`` -- Ignored: ``badge_number, job_title, most_recent_salary, unique_identifier`` (Unused but command will not fail when field is present) - -Details: -~~~~~~~~ -- ``department_name`` Name of department exactly as it is in the server database. - This needs to match the department name provided with the command. -- ``department_state`` Name of department state exactly as it is in the server database, which will be the - `standard two-letter abbreviation `_ for the department's respective location. - This needs to match the department state provided with the command. -- ``unique_internal_identifier`` A string or number that can be used to - uniquely identify the officer, in departments in which the badge - number stays with the officer using that number is fine. Can and should be left blank - if no such number is available. -- ``first_name`` & ``last_name`` Will be inserted into the database as is. -- ``middle_initial`` Usually up to one character, but can be more. -- ``suffix`` Choice of ``Jr, Sr, II, III, IV, V``. -- ``gender`` Choice of ``M``, ``F``, ``Other``. -- ``race`` One of ``BLACK``, ``WHITE``, ``ASIAN``, ``HISPANIC``, ``NATIVE AMERICAN``, ``PACIFIC ISLANDER``, ``Other``. -- ``employment_date`` :ref:`Date ` representing the start of employment with this department. -- ``birth_year`` Integer representing the birth year of the officer. - -Assignments csv -^^^^^^^^^^^^^^^ -- Required: ``id, officer_id, job_title`` -- Optional: ``badge_number, unit_id, unit_name, start_date, resign_date`` - -Details: -~~~~~~~~ -- ``officer_id`` Number referring to ``id`` of existing officer or string starting with ``#`` referring to a newly created officer in the provided officers csv. -- ``badge_number`` Any string that represents the star or badge number of the officer. In some departments this number changes with the assignment. -- ``job_title`` The job title, will be created if it does not exist. -- ``unit_id`` Id of existing unit within the department. -- ``unit_name`` Name of the unit, only used if the ``unit_id`` column is not provided. -- ``start_date`` Start :ref:`date ` of this assignment. -- ``resign_date`` End :ref:`date ` of this assignment. - -Special Flag: -~~~~~~~~~~~~~ -The ``--overwrite-assignments`` in the command can be used to not merge new with existing assignments. -Instead all existing assignments belonging to officers named in the ``officer_id`` column are deleted first, -before the new assignments contained in the provided csv are created in the database. - -This should only be used if the provided csv contains both the currently in the database and additional assignments, -or is based on a better and more complete dataset, for example after receiving a dataset for historic assignment data. - -Salaries csv -^^^^^^^^^^^^ -- Required: ``id, officer_id, salary, year`` -- Optional: ``overtime_pay, is_fiscal_year`` - -Details: -~~~~~~~~ -- ``officer_id`` Integer referring to ``id`` of existing officer or string starting with ``#`` referring to a newly created officer in the provided officers csv -- ``salary`` Number representing the officer's salary in the given year. -- ``year`` Integer, the year this salary information refers to. -- ``overtime_pay`` Number representing the amount of overtime payment for offer in given year. -- ``is_fiscal_year`` Boolean value, indicating whether the provided year refers to calendar year or fiscal year. - The values ``true``, ``t``, ``yes`` and ``y`` are treated as "yes, the salary is for the fiscal year", all others (including blank) as "no" - -Incidents csv -^^^^^^^^^^^^^ -- Required: ``id, department_name, department_state`` -- Optional: ``date, time, report_number, description, street_name, cross_street1, cross_street2, city, state, zip_code, - created_by, last_updated_by, officer_ids, license_plates`` - -Details: -~~~~~~~~ -- ``department_name`` Name of department exactly as in the server database. - This needs to match the department name provided with the command. -- ``department_state`` Name of department state exactly as it is in the server database, which will be the - `standard two-letter abbreviation `_ for the department's respective location. -- ``date`` :ref:`Date ` of the incident -- ``time`` :ref:`Time ` of the incident -- ``report_number`` String representing any kind of number assigned to complaints or incidents by the police department. -- ``description`` Text description of the incident. -- ``street_name`` Name of the street the incident occurred, but should not include the street number. -- ``cross_street1``, ``cross_street2`` The two closest intersecting streets. -- ``city``, ``state``, ``zip_code`` State needs to be in 2 letter abbreviated notation. -- ``created_by``, ``last_updated_by`` Id of existing user shown as responsible for adding this entry. -- ``officer_ids`` Ids of officers involved in the incident, separated by ``|``. - - - Each individual id can either be an integer referring to an existing officer or a string starting with ``#`` referring to a newly created officer. - - Example: ``123|#C1|1627`` for three officers, one with id 123, one with 1627 and one whose record was created in the officers csv - and whose id-field was the string ``#C1``. - -- ``license_plates`` all license plates involved in the incident. If there is more than one, they can be separated with a ``|``. - - - Each license plate consists of the license plate number and optionally a state in abbreviated form separated by an underscore ``_``. - - Example: ``ABC123_IL|B991`` for one license plate with number ``ABC123`` from Illinois and one with number ``B991`` and no associated state. - - -Links csv -^^^^^^^^^ -- Required: ``id, url`` -- Optional: ``title, link_type, description, author, created_by, officer_ids, incident_ids`` - -Details: -~~~~~~~~ -- ``url`` Full url of the link starting with ``http://`` or ``https://``. -- ``title`` Text that will be displayed as the link. -- ``description`` A short description of the link. -- ``link_type`` Choice of ``Link``, ``YouTube Video`` and ``Other Video``. -- ``author`` The source or author of the linked article, report, video. -- ``created_by`` Id of existing user shown as responsible for adding this entry. -- ``officer_ids`` Ids of officer profiles this link should be visible on, separated by ``|``. See same field in incidents above for more details. -- ``incidents_ids`` Ids of incidents this link should be associated with, separated by ``|``. Just like ``officer_ids`` this can contain strings - starting with ``#`` to refer to an incident created in the incident csv. - -Examples ---------- -Example csvs can be found in the repository under ``OpenOversight/tests/test_csvs``. - -.. _ref-aci-force-create: - -Local development flag ``--force-create`` ------------------------------------------ -This flag chances the behavior when an integer is provided as ``id``. Instead of updating an existing record, -a new record will be created and assigned the given ``id``. If a record with that ``id`` already exists in the -database, it will be deleted before the new record is created. - -This functionality is intended to be used to import csv files downloaded from `OpenOversight download page `_ -to get a local copy of the production data for one department in the local development database. diff --git a/docs/area_coordinator.rst b/docs/area_coordinator.md similarity index 100% rename from docs/area_coordinator.rst rename to docs/area_coordinator.md diff --git a/docs/bulk_upload.md b/docs/bulk_upload.md new file mode 100644 index 000000000..27d9c0cf2 --- /dev/null +++ b/docs/bulk_upload.md @@ -0,0 +1,85 @@ +Using the bulk upload feature +============================= +To facilitate adding rosters of new departments or updating existing +departments there is a flask command that gives system administrators +the ability to load the data from a csv file directly into the database. + +**Warning** This process is not very robust at this point and there is +risk of leaving the database in an inconsistent state. It is strongly +recommended to back up the database before starting this operation and +to run the command on a development server first and see if the results +are as expected. + +Preparation steps +----------------- +- Create department if it does not exist +- Add ranks in hierarchical order, make sure all the ranks present in + the csv are added + +Layout of the csv file +---------------------- +The csv file can have the following fields: +``` + department_id + unique_internal_identifier + first_name + last_name + middle_initial + suffix + gender + race + employment_date + birth_year + star_no + job_title + unit_id + start_date + resign_date + salary + salary_year + salary_is_fiscal_year + overtime_pay +``` + + +Explanation of the individual fields +------------------------------------ +General information: +- `department_id` - ID of department in the server database, for example `1` for Chicago Police Department can be found in + url for that department: `https://openoversight.com/departments/1`. +- `unique_internal_identifier` - A string or number that can be used to uniquely identify the officer, in departments in + which the badge number stays with the officer using that number is fine, otherwise it is recommended to leave this + blank and provide the `star_no` instead. +- `first_name` & `last_name` & `middle_initial` +- `suffix` - One of the following values: `Jr, Sr, II, III, IV, V`. +- `gender` - One of the following values: `M`, `F`, `Other` or `Not Sure`. +- `race` - One of the following values: `BLACK`, `WHITE`, `ASIAN`, `HISPANIC`, `NATIVE AMERICAN`, `PACIFIC ISLANDER`, `Other`, `Not Sure`. +- `employment_date` - Start of employment with this department. +- `birth_year` - Integer representing the birth year of the officer in a `yyyy` format + +Current Employment information: +- `star_no` - Star or badge number, might be related to current rank. +- `job_title` - Rank or title, needs to be added to this department verbatim or `Not Sure`, +- `unit_id` - ID of unit within the department. +- `start_date` - Start date of this assignment. +- `resign_date` - Resignation date of this assignment. + +Salary information: +- `salary_year` - Year of which the salary information is provided in a `yyyy` format. +- `salary_is_fiscal_year` - `true` or `false`, salary information is on fiscal year basis vs. calendar year. +- `salary` - Salary in given year. +- `overtime_pay` - Overtime received in given year. + +Required fields +- `department_id`, `first_name`, `last_name`, `job_title` and either `star_no` or `unique_internal_identifier` are required. +- `employment_date`, `start_date` and `resign_date` can be either in `yyyy-mm-dd` or `mm/dd/yyyy` format - if the column is present + the field cannot be left blank. + +Command-line options +- `--no-create` - For each line in the CSV, update an existing officer if one exists, but do not create any new officers. If an officer in the CSV is not already in OpenOversight, the line will be ignored. +- `--update-by-name` - Update officers by `first_name` and `last_name`. Useful when `unique_internal_identifier` and `star_no` are not available. +- `--update-static-fields` - Allow modifications to normally-static fields like `race`, `birth_year`, etc., which OpenOversight normally prevents from being modified. Values in the database will be overwritten with values in the CSV. + +The command to run on the server +-------------------------------- +`/usr/src/app/OpenOversight$ flask bulk-add-officers [/path/to/csv_file.csv]` diff --git a/docs/bulk_upload.rst b/docs/bulk_upload.rst deleted file mode 100644 index e7ceb40dc..000000000 --- a/docs/bulk_upload.rst +++ /dev/null @@ -1,108 +0,0 @@ -Using the bulk upload feature -============================= - -To facilitate adding rosters of new departments or updating existing -departments there is a flask command that gives system administrators -the ability to load the data from a csv file directly into the database. - -**Warning** This process is not very robust at this point and there is -risk of leaving the database in an inconsistent state. It is strongly -recommended to back up the database before starting this operation and -to run the command on a development server first and see if the results -are as expected. - -Preparation steps ------------------ - -- Create department if it does not exist -- Add ranks in hierarchical order, make sure all the ranks present in - the csv are added - -Layout of the csv file ----------------------- - -The csv file can have the following fields: - -:: - - department_id - unique_internal_identifier - first_name - last_name - middle_initial - suffix - gender - race - employment_date - birth_year - star_no - job_title - unit_id - start_date - resign_date - salary - salary_year - salary_is_fiscal_year - overtime_pay - -Explanation of the individual fields ------------------------------------- - -General information: -~~~~~~~~~~~~~~~~~~~~ - -- ``department_id`` id of department in the server database, for - example ``1`` for Chicago Police Department can be found in url for - that department: https://openoversight.com/department/1 -- ``unique_internal_identifier`` a string or number that can be used to - uniquely identify the officer, in departments in which the badge - number stays with the officer using that number is fine, otherwise it - is recommended to leave this blank and provide the ``star_no`` - instead. -- ``first_name`` & ``last_name`` & ``middle_initial`` -- ``suffix`` one of ``Jr, Sr, II, III, IV, V`` -- ``gender`` one of ``M``, ``F``, ``Other`` or ``Not Sure`` -- ``race`` one of ``BLACK``, ``WHITE``, ``ASIAN``, ``HISPANIC``, - ``NATIVE AMERICAN``, ``PACIFIC ISLANDER``, ``Other``, ``Not Sure`` -- ``employment_date`` start of employment with this department -- ``birth_year`` - -Current Employment information: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- ``star_no`` star or badge number, might be related to current rank -- ``job_title`` rank or title, needs to be added to this department - verbatim or ``Not Sure`` -- ``unit_id`` id of unit within the department -- ``start_date`` start date of this assignment -- ``resign_date`` resignation date of this assignment - -Salary information: -~~~~~~~~~~~~~~~~~~~ - -- ``salary_year`` year of which the salary information is provided -- ``salary_is_fiscal_year`` 'true' or 'false', salary information is on - fiscal year basis vs. calendar year -- ``salary`` salary in given year -- ``overtime_pay`` overtime received in given year - -Required fields -~~~~~~~~~~~~~~~ - -- ``department_id``, ``first_name``, ``last_name``, ``job_title`` and - either ``star_no`` or ``unique_internal_identifier`` are required. -- ``employment_date``, ``start_date`` and ``resign_date`` can be either - in ``yyyy-mm-dd`` or ``mm/dd/yyyy`` format - if the column is present - the field cannot be left blank - -Command-line options --------------------- - -- ``--no-create`` - For each line in the CSV, update an existing officer if one exists, but do not create any new officers. If an officer in the CSV is not already in OpenOversight, the line will be ignored. -- ``--update-by-name`` - Update officers by ``first_name`` and ``last_name``. Useful when ``unique_internal_identifier`` and ``star_no`` are not available. -- ``--update-static-fields`` - Allow modifications to normally-static fields like ``race``, ``birth_year``, etc, which OpenOversight normally prevents from being modified. Values in the database will be overwritten with values in the CSV. - -The command to run on the server --------------------------------- - -``/usr/src/app/OpenOversight$ flask bulk-add-officers [/path/to/csv_file.csv]`` diff --git a/docs/conf.py b/docs/conf.py index 4c042b6c9..a1b3abfc2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,7 @@ from typing import List, Tuple +from recommonmark.parser import CommonMarkParser + # # Configuration file for the Sphinx documentation builder. @@ -7,18 +9,6 @@ # This file does only contain a selection of the most common options. For a # full list see the documentation: # http://www.sphinx-doc.org/en/master/config - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - - # -- Project information ----------------------------------------------------- project = "OpenOversight" @@ -40,7 +30,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions: List[str] = [] +extensions = ["recommonmark"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -48,8 +38,14 @@ # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -# source_suffix = ['.rst', '.md'] -source_suffix = ".rst" +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", +} + +source_parsers = { + ".md": CommonMarkParser, +} # The master toctree document. master_doc = "index" @@ -59,7 +55,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -86,7 +82,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +html_static_path = ["img"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -104,22 +100,6 @@ # Output file base name for HTML help builder. htmlhelp_basename = "OpenOversightdoc" - -# -- Options for LaTeX output ------------------------------------------------ -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents: List[Tuple[str, str, str, str, str]] = [ - ( - master_doc, - "OpenOversight.tex", - "OpenOversight Documentation", - "Lucy Parsons Labs", - "manual", - ), -] - - # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples diff --git a/requirements.txt b/requirements.txt index 95383d7c1..fe2357a36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,12 +53,13 @@ pytest~=7.4.0 python-dateutil==2.8.2 pytz==2023.3 PyYAML~=6.0 +recommonmark==0.7.1 requests~=2.31.0 rich~=13.4.2 s3transfer~=0.6.1 selenium~=4.10.0 six~=1.16.0 -sphinx~=7.0.1 +sphinx==7.1.2 SQLAlchemy==1.4.47 # Updating this breaks the build for python 3.9 tornado~=6.3.2 trio==0.22.1