From e554e21b95289aecc65d6c81a6fab97dc38770f0 Mon Sep 17 00:00:00 2001 From: Ben Boger Date: Tue, 23 Jul 2024 14:52:27 -0600 Subject: [PATCH 1/3] adding support for including a get_last_insert_id this enables us to pass a get_last_insert_id to a method with a @sqlupdate annotation defined as a kwarg. --- README.rst | 34 ++++++++++++++-- dysql/connections.py | 54 ++++++++++++++++++------- dysql/test/test_sql_insert_templates.py | 35 +++++++++++++++- 3 files changed, 105 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index 21939e0..5a1e7e8 100644 --- a/README.rst +++ b/README.rst @@ -406,9 +406,9 @@ query returning a dictionary where there are multiple results under each key. No def get_status_by_name(): return QueryData("SELECT status, name FROM table") - +========== @sqlupdate -~~~~~~~~~~ +========== Handles any SQL that is not a select. This is primarily, but not limited to, ``insert``, ``update``, and ``delete``. @@ -418,6 +418,10 @@ Handles any SQL that is not a select. This is primarily, but not limited to, ``i def insert_items(item_dict): return QueryData("INSERT INTO", template_params={'in__item_id':item_id_list}) + +--------------------------------- +multiple queries in a transaction +--------------------------------- You can yield multiple QueryData objects. This is done in a transaction and it can be helpful for data integrity or just a nice clean way to run a set of updates. @@ -430,7 +434,12 @@ a nice clean way to run a set of updates. yield QueryData(f'INSERT INTO table_1 {insert_values_1}', query_params=insert_values_params_1) yield QueryData(f'INSERT INTO table_2 {insert_values_2}', query_params=insert_values_params_2) -if needed you can assign a callback to be ran after a query or set of queries completes successfully +-------------------------- +getting the last insert id +-------------------------- +You can assign a callback to be ran after a query or set of queries completes successfully. This is useful when you need +to get the last insert id for a table that has an auto incrementing id field. This allows you to set it as a parameter on +a follow up relational table within the same transaction scope. .. code-block:: python @@ -444,6 +453,25 @@ if needed you can assign a callback to be ran after a query or set of queries co def _handle_insert_success(item_dict): # callback logic here happens after the transaction is complete +In order to do this, set use_get_last_insert_id=True in the sqludate decorator, and add a get_last_insert_id method as a kwarg to your +function. This method will be overwritten by the sqlupdate decorator so just set it equal to None. + + +.. note:: + You should always define the get_last_insert_id method as a kwarg equal None in the function signature. This is because + the sqlupdate decorator will overwrite the kwarg with the correct method to get the last insert id. + This is only available for the ``sqlupdate`` decorator + +.. code-block:: python + + @sqlupdate(use_get_last_insert_id=True) + def insert_item_with_get_last_insert(get_last_insert_id=None, item_dict): + insert_values, insert_params = TemplateGenerator.values('table1values', _get_values_from_items(item_dict)) + yield QueryData(f'INSERT INTO table_1 {insert_values}', query_params=insert_values_params) + last_id = get_last_insert_id() + yield QueryData(f'INSERT INTO related_table_1 (table_1_id, value) VALUES (:table_1_id, :value)', + query_params={'table_1_id': last_id, 'value': 'some_value'}) + @sqlexists ~~~~~~~~~~ This wraps a SQL query to determine if a row exists or not. If at least one row is returned from the query, it will diff --git a/dysql/connections.py b/dysql/connections.py index 29ee2ac..9ad7638 100644 --- a/dysql/connections.py +++ b/dysql/connections.py @@ -20,10 +20,8 @@ ) from .query_utils import get_query_data - logger = logging.getLogger("database") - # Always initialize a database container, it is never set again _DATABASE_CONTAINER = DatabaseContainerSingleton() @@ -126,7 +124,7 @@ def handle_query(*args, **kwargs): actual_mapper = actual_mapper() with _ConnectionManager( - func, isolation_level, False, *args, **kwargs + func, isolation_level, False, *args, **kwargs ) as conn_manager: data = func(*args, **kwargs) query, params = get_query_data(data) @@ -163,7 +161,7 @@ def decorator(func): def handle_query(*args, **kwargs): functools.wraps(func, handle_query) with _ConnectionManager( - func, isolation_level, False, *args, **kwargs + func, isolation_level, False, *args, **kwargs ) as conn_manager: data = func(*args, **kwargs) query, params = get_query_data(data) @@ -182,7 +180,10 @@ def handle_query(*args, **kwargs): def sqlupdate( - isolation_level="READ_COMMITTED", disable_foreign_key_checks=False, on_success=None + isolation_level="READ_COMMITTED", + disable_foreign_key_checks=False, + on_success=None, + use_get_last_insert_id=False ): """ :param isolation_level should specify whether we can read data from transactions that are not @@ -200,16 +201,36 @@ def sqlupdate( Note: this will work as expected when a normal transaction completes successfully and if a transaction rolls back this will be left in a clean state as expected before executing anything - Examples:: + :param use_get_last_insert_id: If set to True, the function will be added to the kwargs and passed + into the function we are calling. This will allow us to get the last insert id from the database + within the transaction scope of a give sql update method - @sqlinsert - def insert_example(key_values) - return "INSERT INTO table(id, value) VALUES (:id, :value)", key_values + This could be particularly helpful when you are inserting into a table and + want to follow up with another insert that requires the last insert id or when + you might want to use the last insert id in the object passed in. - @sqlinsert - def delete_example(ids) - return "DELETE FROM table", key_values + Note: This will only give you the id of the last record so if you insert multiple values you will only + be able to get the last id inserted out of all the values inserted. if you needed the id for each value + you were inserting you would need to yield insert each row and get_last_insert_id after each insert. + Examples:: + + @sqlupdate + def insert_example(key_values) + return QueryData("INSERT INTO table(id, value) VALUES (:id, :value)", key_values) + + @sqlupdate + def delete_example(ids) + return QueryData("DELETE FROM table WHERE id=:id", { "id": id }) + + @sqlupdate(use_get_last_insert_id=True) + def insert_with_relations(get_last_insert_id = None): + yield QueryData("INSERT INTO table(value) VALUES (:value)", key_values) + id = get_last_insert_id() + yield "INSERT INTO relation_table(id, value) VALUES (:id, :value)", { + "id": id, + "value": "value" + }) """ def update_wrapper(func): @@ -221,11 +242,13 @@ def update_wrapper(func): def handle_query(*args, **kwargs): functools.wraps(func) with _ConnectionManager( - func, isolation_level, True, *args, **kwargs + func, isolation_level, True, *args, **kwargs ) as conn_manager: if disable_foreign_key_checks: conn_manager.execute_query("SET FOREIGN_KEY_CHECKS=0") - + last_insert_method = 'get_last_insert_id' + if use_get_last_insert_id: + kwargs[last_insert_method] = lambda: conn_manager.execute_query("SELECT LAST_INSERT_ID()").scalar() if inspect.isgeneratorfunction(func): logger.debug("handling each query before committing transaction") @@ -245,6 +268,9 @@ def handle_query(*args, **kwargs): if disable_foreign_key_checks: conn_manager.execute_query("SET FOREIGN_KEY_CHECKS=1") + + if last_insert_method in kwargs: + del kwargs[last_insert_method] if on_success: on_success(*args, **kwargs) diff --git a/dysql/test/test_sql_insert_templates.py b/dysql/test/test_sql_insert_templates.py index c38986c..b101aeb 100644 --- a/dysql/test/test_sql_insert_templates.py +++ b/dysql/test/test_sql_insert_templates.py @@ -22,8 +22,18 @@ @pytest.fixture(name="mock_engine", autouse=True) def mock_engine_fixture(mock_create_engine): + + initial_id = 0 + def handle_execute(query = None, args = None): + nonlocal initial_id + if "INSERT INTO get_last(name)" in query.text: + initial_id += 1 + if "SELECT LAST_INSERT_ID()" == query.text: + return type("Result", (), {"scalar": lambda: initial_id}) + return [] mock_engine = setup_mock_engine(mock_create_engine) - mock_engine.connect().execution_options().execute.side_effect = lambda x, y: [] + execute_mock = mock_engine.connect().execution_options().execute + execute_mock.side_effect = handle_execute return mock_engine @@ -140,3 +150,26 @@ def insert_into_single_value(names): return QueryData( "INSERT INTO table(name) {values__name_col}", template_params=template_params ) + + +def test_last_insert_id(): + @sqlupdate(use_get_last_insert_id=True) + def insert(get_last_insert_id = None): + yield QueryData("INSERT INTO get_last(name) VALUES ('Tom')") + assert get_last_insert_id + assert get_last_insert_id() == 1 + yield QueryData("INSERT INTO get_last(name) VALUES ('Jerry')") + assert get_last_insert_id() == 2 + insert() + +def test_last_insert_id_removed_before_callback(): + def callback(**kwargs): + assert "get_last_insert_id" not in kwargs + + @sqlupdate(use_get_last_insert_id=True,) + def insert( get_last_insert_id = None): + assert get_last_insert_id + yield QueryData("INSERT INTO get_last(name) VALUES ('Tom')") + yield QueryData("INSERT INTO get_last(name) VALUES ('Jerry')") + assert get_last_insert_id() == 2 + insert() From 1273ef10f3b76d54e8631819fac9773c1a4e2014 Mon Sep 17 00:00:00 2001 From: Ben Boger Date: Tue, 23 Jul 2024 15:25:53 -0600 Subject: [PATCH 2/3] fixingl ruff formatting --- dysql/connections.py | 14 ++++++++------ dysql/test/test_sql_insert_templates.py | 16 +++++++++++----- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/dysql/connections.py b/dysql/connections.py index 9ad7638..807cf99 100644 --- a/dysql/connections.py +++ b/dysql/connections.py @@ -124,7 +124,7 @@ def handle_query(*args, **kwargs): actual_mapper = actual_mapper() with _ConnectionManager( - func, isolation_level, False, *args, **kwargs + func, isolation_level, False, *args, **kwargs ) as conn_manager: data = func(*args, **kwargs) query, params = get_query_data(data) @@ -161,7 +161,7 @@ def decorator(func): def handle_query(*args, **kwargs): functools.wraps(func, handle_query) with _ConnectionManager( - func, isolation_level, False, *args, **kwargs + func, isolation_level, False, *args, **kwargs ) as conn_manager: data = func(*args, **kwargs) query, params = get_query_data(data) @@ -183,7 +183,7 @@ def sqlupdate( isolation_level="READ_COMMITTED", disable_foreign_key_checks=False, on_success=None, - use_get_last_insert_id=False + use_get_last_insert_id=False, ): """ :param isolation_level should specify whether we can read data from transactions that are not @@ -242,13 +242,15 @@ def update_wrapper(func): def handle_query(*args, **kwargs): functools.wraps(func) with _ConnectionManager( - func, isolation_level, True, *args, **kwargs + func, isolation_level, True, *args, **kwargs ) as conn_manager: if disable_foreign_key_checks: conn_manager.execute_query("SET FOREIGN_KEY_CHECKS=0") - last_insert_method = 'get_last_insert_id' + last_insert_method = "get_last_insert_id" if use_get_last_insert_id: - kwargs[last_insert_method] = lambda: conn_manager.execute_query("SELECT LAST_INSERT_ID()").scalar() + kwargs[last_insert_method] = lambda: conn_manager.execute_query( + "SELECT LAST_INSERT_ID()" + ).scalar() if inspect.isgeneratorfunction(func): logger.debug("handling each query before committing transaction") diff --git a/dysql/test/test_sql_insert_templates.py b/dysql/test/test_sql_insert_templates.py index b101aeb..9ea84af 100644 --- a/dysql/test/test_sql_insert_templates.py +++ b/dysql/test/test_sql_insert_templates.py @@ -22,15 +22,16 @@ @pytest.fixture(name="mock_engine", autouse=True) def mock_engine_fixture(mock_create_engine): - initial_id = 0 - def handle_execute(query = None, args = None): + + def handle_execute(query=None, args=None): nonlocal initial_id if "INSERT INTO get_last(name)" in query.text: initial_id += 1 if "SELECT LAST_INSERT_ID()" == query.text: return type("Result", (), {"scalar": lambda: initial_id}) return [] + mock_engine = setup_mock_engine(mock_create_engine) execute_mock = mock_engine.connect().execution_options().execute execute_mock.side_effect = handle_execute @@ -154,22 +155,27 @@ def insert_into_single_value(names): def test_last_insert_id(): @sqlupdate(use_get_last_insert_id=True) - def insert(get_last_insert_id = None): + def insert(get_last_insert_id=None): yield QueryData("INSERT INTO get_last(name) VALUES ('Tom')") assert get_last_insert_id assert get_last_insert_id() == 1 yield QueryData("INSERT INTO get_last(name) VALUES ('Jerry')") assert get_last_insert_id() == 2 + insert() + def test_last_insert_id_removed_before_callback(): def callback(**kwargs): assert "get_last_insert_id" not in kwargs - @sqlupdate(use_get_last_insert_id=True,) - def insert( get_last_insert_id = None): + @sqlupdate( + use_get_last_insert_id=True, + ) + def insert(get_last_insert_id=None): assert get_last_insert_id yield QueryData("INSERT INTO get_last(name) VALUES ('Tom')") yield QueryData("INSERT INTO get_last(name) VALUES ('Jerry')") assert get_last_insert_id() == 2 + insert() From 839bb584e6d97b031f47dca321898dfdf9471f22 Mon Sep 17 00:00:00 2001 From: Ben Boger Date: Tue, 23 Jul 2024 16:46:25 -0600 Subject: [PATCH 3/3] updating to inspect the signature this removes the need to have 2 parameters in 2 places now you only need to define the get_last_insert_id=None --- README.rst | 33 +++++++++++++++++++------ dysql/connections.py | 17 ++----------- dysql/test/test_sql_insert_templates.py | 6 ++--- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/README.rst b/README.rst index 5a1e7e8..469bf19 100644 --- a/README.rst +++ b/README.rst @@ -443,7 +443,7 @@ a follow up relational table within the same transaction scope. .. code-block:: python - @sqlupdate(on_success=_handle_insert_success) + @sqlupdate() def insert_items_with_callback(item_dict): insert_values_1, insert_params_1 = TemplateGenerator.values('table1values', _get_values_for_1_from_items(item_dict)) insert_values_2, insert_params_2 = TemplateGenerator.values('table2values', _get_values_for_2_from_items(item_dict)) @@ -453,18 +453,17 @@ a follow up relational table within the same transaction scope. def _handle_insert_success(item_dict): # callback logic here happens after the transaction is complete -In order to do this, set use_get_last_insert_id=True in the sqludate decorator, and add a get_last_insert_id method as a kwarg to your -function. This method will be overwritten by the sqlupdate decorator so just set it equal to None. +`get_last_insert_id` is a placeholder kwarg that will be automatically overwritten by the sqlupdate decorator at run time. +Therefore, the assigned value in the function definition does not matter. -.. note:: - You should always define the get_last_insert_id method as a kwarg equal None in the function signature. This is because - the sqlupdate decorator will overwrite the kwarg with the correct method to get the last insert id. - This is only available for the ``sqlupdate`` decorator +Using `get_last_insert_id` gives you the most recently set id. You can leverage this for later queries yielded, or you could +use it and set ids in a reference object passed in for access to the ides outside of the sqlupdate function. + .. code-block:: python - @sqlupdate(use_get_last_insert_id=True) + @sqlupdate() def insert_item_with_get_last_insert(get_last_insert_id=None, item_dict): insert_values, insert_params = TemplateGenerator.values('table1values', _get_values_from_items(item_dict)) yield QueryData(f'INSERT INTO table_1 {insert_values}', query_params=insert_values_params) @@ -472,6 +471,24 @@ function. This method will be overwritten by the sqlupdate decorator so just set yield QueryData(f'INSERT INTO related_table_1 (table_1_id, value) VALUES (:table_1_id, :value)', query_params={'table_1_id': last_id, 'value': 'some_value'}) +.. note:: + `get_last_insert_id` will get you the last inserted id from the most recently table inserted with an autoincrement. + Be sure to call `get_last_insert_id` right after you yield the query that inserts the record you need the id for. + + +.. code-block:: python + + class Item(BaseModel): + id: int | None = None + name: str + + @sqlupdate() + def insert_items_and_update_ids(items: List[Item], get_last_insert_id = None) + for item in items: + yield QueryData("INSERT INTO table (name) VALUES (:name)", query_params={'name': item.name}) + last_id = get_last_insert_id() + item.id = last_id + @sqlexists ~~~~~~~~~~ This wraps a SQL query to determine if a row exists or not. If at least one row is returned from the query, it will diff --git a/dysql/connections.py b/dysql/connections.py index 807cf99..53379c0 100644 --- a/dysql/connections.py +++ b/dysql/connections.py @@ -183,7 +183,6 @@ def sqlupdate( isolation_level="READ_COMMITTED", disable_foreign_key_checks=False, on_success=None, - use_get_last_insert_id=False, ): """ :param isolation_level should specify whether we can read data from transactions that are not @@ -201,18 +200,6 @@ def sqlupdate( Note: this will work as expected when a normal transaction completes successfully and if a transaction rolls back this will be left in a clean state as expected before executing anything - :param use_get_last_insert_id: If set to True, the function will be added to the kwargs and passed - into the function we are calling. This will allow us to get the last insert id from the database - within the transaction scope of a give sql update method - - This could be particularly helpful when you are inserting into a table and - want to follow up with another insert that requires the last insert id or when - you might want to use the last insert id in the object passed in. - - Note: This will only give you the id of the last record so if you insert multiple values you will only - be able to get the last id inserted out of all the values inserted. if you needed the id for each value - you were inserting you would need to yield insert each row and get_last_insert_id after each insert. - Examples:: @sqlupdate @@ -223,7 +210,7 @@ def insert_example(key_values) def delete_example(ids) return QueryData("DELETE FROM table WHERE id=:id", { "id": id }) - @sqlupdate(use_get_last_insert_id=True) + @sqlupdate() def insert_with_relations(get_last_insert_id = None): yield QueryData("INSERT INTO table(value) VALUES (:value)", key_values) id = get_last_insert_id() @@ -247,7 +234,7 @@ def handle_query(*args, **kwargs): if disable_foreign_key_checks: conn_manager.execute_query("SET FOREIGN_KEY_CHECKS=0") last_insert_method = "get_last_insert_id" - if use_get_last_insert_id: + if last_insert_method in inspect.signature(func).parameters: kwargs[last_insert_method] = lambda: conn_manager.execute_query( "SELECT LAST_INSERT_ID()" ).scalar() diff --git a/dysql/test/test_sql_insert_templates.py b/dysql/test/test_sql_insert_templates.py index 9ea84af..f50bf86 100644 --- a/dysql/test/test_sql_insert_templates.py +++ b/dysql/test/test_sql_insert_templates.py @@ -154,7 +154,7 @@ def insert_into_single_value(names): def test_last_insert_id(): - @sqlupdate(use_get_last_insert_id=True) + @sqlupdate() def insert(get_last_insert_id=None): yield QueryData("INSERT INTO get_last(name) VALUES ('Tom')") assert get_last_insert_id @@ -169,9 +169,7 @@ def test_last_insert_id_removed_before_callback(): def callback(**kwargs): assert "get_last_insert_id" not in kwargs - @sqlupdate( - use_get_last_insert_id=True, - ) + @sqlupdate() def insert(get_last_insert_id=None): assert get_last_insert_id yield QueryData("INSERT INTO get_last(name) VALUES ('Tom')")