Skip to content

Commit

Permalink
Improve repair performance in cases where there are a lot of finished…
Browse files Browse the repository at this point in the history
… jobs with dependencies
  • Loading branch information
kraih committed Nov 20, 2023
1 parent 8e30c60 commit d2ab280
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 11 deletions.
15 changes: 6 additions & 9 deletions src/pg-backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -348,18 +348,15 @@ export class PgBackend {
DELETE FROM minion_workers WHERE notified < NOW() - INTERVAL '1 millisecond' * ${minion.missingAfter}
`;

// Old jobs with no unresolved dependencies and expired jobs
// Old jobs
await pg.query`
DELETE FROM minion_jobs WHERE id IN (
SELECT j.id FROM minion_jobs AS j LEFT JOIN minion_jobs AS children
ON children.state != 'finished' AND ARRAY_LENGTH(children.parents, 1) > 0 AND j.id = ANY(children.parents)
WHERE j.state = 'finished' AND j.finished <= NOW() - INTERVAL '1 millisecond' * ${minion.removeAfter}
AND children.id IS NULL
UNION ALL
SELECT id FROM minion_jobs WHERE state = 'inactive' AND expires <= NOW()
)
DELETE FROM minion_jobs
WHERE state = 'finished' AND finished <= NOW() - INTERVAL '1 millisecond' * ${minion.removeAfter}
`;

// Expired jobs
await pg.query`DELETE FROM minion_jobs WHERE state = 'inactive' AND expires <= NOW()`;

// Jobs with missing worker (can be retried)
const jobs = await pg.query<JobWithMissingWorkerResult>`
SELECT id, retries FROM minion_jobs AS j
Expand Down
4 changes: 2 additions & 2 deletions test/pg.js
Original file line number Diff line number Diff line change
Expand Up @@ -1027,9 +1027,9 @@ t.test('PostgreSQL backend', skip, async t => {
t.same((await job4.info()).parents, [id, id2]);
t.equal((await minion.stats()).finished_jobs, 2);
await minion.repair();
t.equal((await minion.stats()).finished_jobs, 2);
t.equal((await minion.stats()).finished_jobs, 0);
t.ok(await job4.finish());
t.equal((await minion.stats()).finished_jobs, 3);
t.equal((await minion.stats()).finished_jobs, 1);
await minion.repair();
t.equal((await minion.stats()).finished_jobs, 0);

Expand Down

0 comments on commit d2ab280

Please sign in to comment.