Skip to content

Commit

Permalink
LIMS-276: Speed up count of _get_all_containers results by removing u…
Browse files Browse the repository at this point in the history 
…nnecessary joins (#561)

* LIMS-276: Speed up count of _get_all_containers results by removing unnecessary joins

* Move join functionality to database query builder

* Forgot to commit the shipment change

---------

Co-authored-by: Mark Williams <[email protected]>
Co-authored-by: John Holt <[email protected]>
  • Loading branch information
@ndg63276
3 people authored Aug 23, 2023
1 parent 9c43a25 commit 1809358
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 16 deletions.
32 changes: 31 additions & 1 deletion api/src/Database/DatabaseQueryBuilder.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,11 @@ class DatabaseQueryBuilder
private $query_bound_values = [];
/** @var ?string where clause column name */
private $where_columnName = Null;
/** @var ?mixed where vlause value */
/** @var ?mixed where clause value */
private $where_value = Null;
/** @var array join clauses*/
private $join_clauses = [];

/**
* @var DatabaseParent database parent to use
*/
Expand Down Expand Up @@ -104,6 +107,33 @@ public function insert($expectedTable)
return $this->db->id();
}


/**
* Add while join clauses with no processing except to remove duplicates
* ONLY USED WITH getJoins at the moment
*
* @param string $join_clause The join clause to add, e.g. JOIN table t ON t.id = s.id
* @return DatabaseQueryBuilder the builder
*/
public function joinClause($join_clause)
{
if (!in_array($join_clause, $this->join_clauses))
array_push($this->join_clauses, $join_clause);
return $this;
}

/**
* Pull back the join clauses ready to be added to a SQL statement
*
* This is a half-way house into adding join caluses to this builder
*
* @return string The join caluses as one long string
*/
public function getJoins()
{
return implode(" ", $this->join_clauses);
}

private function bindArrayAsList($values): string
{
$bound_list_sql = "";
Expand Down
42 changes: 27 additions & 15 deletions api/src/Page/Shipment.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
namespace SynchWeb\Page;

use Exception;
use SynchWeb\Database\DatabaseQueryBuilder;
use SynchWeb\Page;
use SynchWeb\Shipment\Courier\DHL;
use SynchWeb\Shipment\ShippingService;
Expand Down Expand Up @@ -1644,20 +1645,24 @@ function _check_container()
$this->_output($cont[0]);
}



function _get_all_containers()
{
//$this->db->set_debug(True);
if (!$this->has_arg('prop') && !$this->has_arg('visit') && !$this->staff)
$this->_error('No proposal specified');

$having = '';
$subsamplesInTotal = "";
$totalQuery = new DatabaseQueryBuilder($this->db);

if ($this->has_arg('visit')) {
$join = " INNER JOIN blsession ses2 ON ses2.proposalid = p.proposalid";
$args = array($this->arg('visit'));
$where = "CONCAT(p.proposalcode, p.proposalnumber, '-', ses2.visit_number) LIKE :1";
$totalQuery->joinClause("INNER JOIN blsession ses2 ON ses2.proposalid = p.proposalid");
$totalQuery->joinClause("INNER JOIN dewar d ON d.dewarid = c.dewarid");
$totalQuery->joinClause("INNER JOIN shipping sh ON sh.shippingid = d.shippingid");
$totalQuery->joinClause("INNER JOIN proposal p ON p.proposalid = sh.proposalid");
} else if ($this->has_arg('all') && $this->staff) {
$join = '';
$args = array();
Expand All @@ -1666,6 +1671,8 @@ function _get_all_containers()
$join = '';
$args = array($this->proposalid);
$where = 'sh.proposalid=:1';
$totalQuery->joinClause("INNER JOIN dewar d ON d.dewarid = c.dewarid");
$totalQuery->joinClause("INNER JOIN shipping sh ON sh.shippingid = d.shippingid");
}


Expand All @@ -1679,16 +1686,22 @@ function _get_all_containers()
} else if ($this->arg('ty') == 'todispose') {
$where .= " AND c.imagerid IS NOT NULL";
$having .= " HAVING (TIMESTAMPDIFF('HOUR', min(ci.bltimestamp), CURRENT_TIMESTAMP)/24) > 42";
$totalQuery->joinClause("LEFT OUTER JOIN containerinspection ci ON ci.containerid = c.containerid AND ci.state = 'Completed'");
} else if ($this->arg('ty') == 'queued') {
$where .= " AND cq.containerqueueid IS NOT NULL";
$totalQuery->joinClause("LEFT OUTER JOIN containerqueue cq ON cq.containerid = c.containerid AND cq.completedtimestamp IS NULL");
} else if ($this->arg('ty') == 'completed') {
$where .= " AND cq2.completedtimestamp IS NOT NULL";
$totalQuery->joinClause("LEFT OUTER JOIN containerqueue cq2 ON cq2.containerid = c.containerid AND cq2.completedtimestamp IS NOT NULL");
$this->args['sort_by'] = 'COMPLETEDTIMESTAMP';
$this->args['order'] = 'desc';
} else if ($this->arg('ty') == 'processing') {
$where .= " AND c.containerstatus = 'processing'";
} else if ($this->arg('ty') == 'subsamples') {
$having .= " HAVING subsamples > 0";
$subsamplesInTotal = ", count(distinct ss.blsubsampleid) as subsamples";
$totalQuery->joinClause("LEFT OUTER JOIN blsample s ON s.containerid = c.containerid");
$totalQuery->joinClause("LEFT OUTER JOIN blsubsample ss ON s.blsampleid = ss.blsampleid AND ss.source='manual'");
}
}

Expand All @@ -1700,10 +1713,13 @@ function _get_all_containers()

if ($this->has_arg('did')) {
$where .= ' AND d.dewarid=:' . (sizeof($args) + 1);
$totalQuery->joinClause("INNER JOIN dewar d ON d.dewarid = c.dewarid");
array_push($args, $this->arg('did'));
}
if ($this->has_arg('sid')) {
$where .= ' AND sh.shippingid=:' . (sizeof($args) + 1);
$totalQuery->joinClause("INNER JOIN dewar d ON d.dewarid = c.dewarid");
$totalQuery->joinClause("INNER JOIN shipping sh ON sh.shippingid = d.shippingid");
array_push($args, $this->arg('sid'));
}
if ($this->has_arg('cid')) {
Expand All @@ -1715,11 +1731,15 @@ function _get_all_containers()
// $this->db->set_debug(True);
$join .= ' LEFT OUTER JOIN crystal cr ON cr.crystalid = s.crystalid LEFT OUTER JOIN protein pr ON pr.proteinid = cr.proteinid';
$where .= ' AND pr.proteinid=:' . (sizeof($args) + 1);
$totalQuery->joinClause("LEFT OUTER JOIN blsample s ON s.containerid = c.containerid");
$totalQuery->joinClause("LEFT OUTER JOIN crystal cr ON cr.crystalid = s.crystalid");
$totalQuery->joinClause("LEFT OUTER JOIN protein pr ON pr.proteinid = cr.proteinid");
array_push($args, $this->arg('pid'));
}

if ($this->has_arg('assigned')) {
$where .= " AND d.dewarstatus LIKE 'processing' AND c.samplechangerlocation > 0";
$totalQuery->joinClause("INNER JOIN dewar d ON d.dewarid = c.dewarid");
}

if ($this->has_arg('bl')) {
Expand All @@ -1731,6 +1751,8 @@ function _get_all_containers()
$where .= " AND c.containerid NOT IN (SELECT c.containerid FROM container c INNER JOIN dewar d ON d.dewarid = c.dewarid WHERE d.dewarstatus LIKE 'processing' AND c.samplechangerlocation > 0 AND c.beamlinelocation=:" . (sizeof($args) + 1) . ")";

array_push($args, $this->arg('unassigned'));
$totalQuery->joinClause("INNER JOIN dewar d ON d.dewarid = c.dewarid");
$totalQuery->joinClause("INNER JOIN shipping sh ON sh.shippingid = d.shippingid");
$this->args['sort_by'] = 'SHIPPINGID';
$this->args['order'] = 'desc';
}
Expand Down Expand Up @@ -1758,20 +1780,10 @@ function _get_all_containers()
array_push($args, $this->user->personId);
}

$tot = $this->db->pq("SELECT count(distinct c.containerid) as tot,
count(distinct ss.blsubsampleid) as subsamples
$tot = $this->db->pq("SELECT count(distinct c.containerid) as tot
$subsamplesInTotal
FROM container c
INNER JOIN dewar d ON d.dewarid = c.dewarid
INNER JOIN shipping sh ON sh.shippingid = d.shippingid
INNER JOIN proposal p ON p.proposalid = sh.proposalid
LEFT OUTER JOIN blsample s ON s.containerid = c.containerid
LEFT OUTER JOIN blsubsample ss ON s.blsampleid = ss.blsampleid AND ss.source='manual'
LEFT OUTER JOIN crystal cr ON cr.crystalid = s.crystalid
LEFT OUTER JOIN protein pr ON pr.proteinid = cr.proteinid
LEFT OUTER JOIN containerinspection ci ON ci.containerid = c.containerid AND ci.state = 'Completed'
LEFT OUTER JOIN containerqueue cq ON cq.containerid = c.containerid AND cq.completedtimestamp IS NULL
LEFT OUTER JOIN containerqueue cq2 ON cq2.containerid = c.containerid AND cq2.completedtimestamp IS NOT NULL
$join
{$totalQuery->getJoins()}
WHERE $where
$having", $args);
$tot = sizeof($tot) ? intval($tot[0]['TOT']) : 0;
Expand Down

0 comments on commit 1809358

Please sign in to comment.