From 7b8495b9f12d6ec9575872a4acabd34ff78181a9 Mon Sep 17 00:00:00 2001 From: Nikunj Date: Fri, 3 Feb 2023 17:47:58 +0530 Subject: [PATCH 01/10] [15.0]Added queue job to process crm profiling --- crm_profiling/__manifest__.py | 2 +- crm_profiling/models/__init__.py | 2 +- crm_profiling/models/crm_segmentation.py | 287 ++++++++++-------- crm_profiling/models/queue_job_batch.py | 9 + crm_profiling/views/crm_segmentation_view.xml | 9 +- 5 files changed, 184 insertions(+), 125 deletions(-) create mode 100644 crm_profiling/models/queue_job_batch.py diff --git a/crm_profiling/__manifest__.py b/crm_profiling/__manifest__.py index 9ce76724..825a1e45 100644 --- a/crm_profiling/__manifest__.py +++ b/crm_profiling/__manifest__.py @@ -19,7 +19,7 @@ """, "author": "OpenERP SA", "website": "https://github.com/OCA/server-tools", - "depends": ["base", "crm", "sale"], + "depends": ["account", "contacts", "queue_job_batch"], "data": [ "security/ir.model.access.csv", "wizard/open_questionnaire_view.xml", diff --git a/crm_profiling/models/__init__.py b/crm_profiling/models/__init__.py index 04c01084..b24cada7 100644 --- a/crm_profiling/models/__init__.py +++ b/crm_profiling/models/__init__.py @@ -1 +1 @@ -from . import crm_profiling, crm_segmentation +from . import crm_profiling, crm_segmentation, queue_job_batch diff --git a/crm_profiling/models/crm_segmentation.py b/crm_profiling/models/crm_segmentation.py index ad55f495..ae53809e 100644 --- a/crm_profiling/models/crm_segmentation.py +++ b/crm_profiling/models/crm_segmentation.py @@ -1,5 +1,8 @@ +from datetime import datetime + from odoo import _, api, fields, models from odoo.exceptions import ValidationError +from odoo.tools.misc import split_every class Segmentation(models.Model): @@ -27,10 +30,16 @@ class Segmentation(models.Model): "partners that doesn't match segmentation criterions", ) state = fields.Selection( - [("not running", "Not Running"), ("running", "Running")], + [ + ("not running", "Not Running"), + ("running", "Running"), + ("stopped", "Stopped"), + ], "Execution Status", readonly=True, default="not running", + store=True, + compute="_compute_selection", ) partner_id = fields.Integer("Max Partner ID processed", default=0) segmentation_line = fields.One2many( @@ -66,24 +75,66 @@ class Segmentation(models.Model): help="Check this box if you want to use this tab as part of the segmentation rule. " "If not checked, the criteria beneath will be ignored", ) + job_batch_ids = fields.One2many("queue.job.batch", "segmentation_id", readonly=True) + last_batch_id = fields.Many2one("queue.job.batch", readonly=True) - def process_continue(self, start=False): + @api.depends( + "last_batch_id", "last_batch_id.job_ids", "last_batch_id.job_ids.state" + ) + def _compute_selection(self): + for rec in self: + jobs = rec.last_batch_id.job_ids.filtered( + lambda x: x.state not in ["done", "cancelled"] + ) + if jobs: + rec.state = "running" + elif rec.last_batch_id.job_ids.filtered(lambda x: x.state in ["cancelled"]): + rec.state = "stopped" + else: + rec.state = "not running" + + def process_continue(self): """ :param self: The current crm.segmentation. :param start: start boolean flag """ + batch = self.last_batch_id + if not batch: + batches = self.job_batch_ids.sort(key=lambda x: x.id, reverse=True) + if batches: + batch = batches[0] + self.last_batch_id = batch.id + jobs = batch.job_ids.filtered(lambda x: x.state in ["cancelled"]) + if jobs: + jobs.requeue() + return True + def process_stop(self): + batch = self.last_batch_id + if not batch: + batches = self.job_batch_ids.sort(key=lambda x: x.id, reverse=True) + if batches: + batch = batches[0] + self.last_batch_id = batch.id + jobs = batch.job_ids.filtered(lambda x: x.state not in ["done", "cancelled"]) + if jobs: + jobs.button_cancelled() + return True + + def process_start(self): + """ + :param self: The current crm.segmentation. + """ partner_obj = self.env["res.partner"] for seg in self: - if start: - if seg["exclusif"]: - self._cr.execute( - """ - DELETE FROM res_partner_res_partner_category_rel - WHERE category_id=%s""", - (seg.categ_id.id,), - ) - partner_obj.invalidate_cache(["category_id"]) + if seg["exclusif"]: + self._cr.execute( + """ + DELETE FROM res_partner_res_partner_category_rel + WHERE category_id=%s""", + (seg.categ_id.id,), + ) + partner_obj.invalidate_cache(["category_id"]) self._cr.execute("select id from res_partner order by id ") partners = [x[0] for x in self._cr.fetchall()] @@ -91,54 +142,16 @@ def process_continue(self, start=False): if seg.sales_purchase_active: to_remove_list = [] lines = self.segmentation_line - - for pid in partners: - if not lines.test(pid): - to_remove_list.append(pid) - for pid in to_remove_list: - partners.remove(pid) - - if seg.profiling_active: - to_remove_list = [] - for pid in partners: - self.env.cr.execute( - """ - SELECT DISTINCT(answer) FROM partner_question_rel - WHERE partner=%s""", - (pid,), + now = datetime.now() + batch_name = "Batch " + now.strftime("%Y-%m-%d %H:%M:%S") + batch = self.env["queue.job.batch"].get_new_batch(batch_name) + for pids in split_every(100, partners): + lines.with_context(job_batch=batch).with_delay().test( + pids, to_remove_list ) - answers_ids = [x[0] for x in self.env.cr.fetchall()] - - if not self.test_prof(pid, answers_ids): - to_remove_list.append(pid) - for pid in to_remove_list: - partners.remove(pid) - for partner in partner_obj.browse(partners): - category_ids = partner.category_id.ids - if seg.categ_id[0].ids not in category_ids: - self._cr.execute( - """ - INSERT INTO res_partner_res_partner_category_rel - (category_id,partner_id) - VALUES (%s,%s) ON CONFLICT DO NOTHING""", - (seg.categ_id.id, partner.id), - ) - partner_obj.invalidate_cache(["category_id"], [partner.id]) - - seg.write({"state": "not running", "partner_id": 0}) + seg.write({"job_batch_ids": [(4, batch.id)], "last_batch_id": batch.id}) return True - def process_stop(self): - return self.write({"state": "not running", "partner_id": 0}) - - def process_start(self): - """ - :param self: The current crm.segmentation. - """ - - self.write({"state": "running", "partner_id": 0}) - return self.process_continue(start=True) - @api.constrains("parent_id") def _check_parent_id(self): if not self._check_recursion(): @@ -237,72 +250,104 @@ class SegmentationLine(models.Model): default="and", ) - def test(self, partner_id): + def test(self, partners, to_remove_list): """ :param self: The current crm.segmentation.line. :param partner_id: The partner object. """ - - expression = { - "<": lambda x, y: x < y, - "=": lambda x, y: x == y, - ">": lambda x, y: x > y, - } - for line in self: - self.env.cr.execute( - """ - SELECT * FROM ir_module_module WHERE name=%s AND state=%s - """, - ("account", "installed"), - ) - - if self.env.cr.fetchone(): - if line["expr_name"] == "sale": - self._cr.execute( - """SELECT SUM(l.price_unit * l.quantity) - FROM account_move_line l, account_move i - WHERE (l.move_id = i.id) AND - i.partner_id = %s AND - i.move_type = 'out_invoice' - """, - (partner_id,), - ) - value = self.env.cr.fetchone()[0] or 0.0 - self.env.cr.execute( - """SELECT SUM(l.price_unit * l.quantity) - FROM account_move_line l, account_move i - WHERE (l.move_id = i.id) - AND i.partner_id = %s - AND i.move_type = 'out_refund' - """, - (partner_id,), - ) - value -= self.env.cr.fetchone()[0] or 0.0 - elif line["expr_name"] == "purchase": - self.env.cr.execute( - """SELECT SUM(l.price_unit * l.quantity) - FROM account_move_line l, account_move i - WHERE (l.move_id = i.id) - AND i.partner_id = %s - AND i.move_type = 'in_invoice' - """, - (partner_id,), - ) - value = self.env.cr.fetchone()[0] or 0.0 - self.env.cr.execute( - """SELECT SUM(l.price_unit * l.quantity) - FROM account_move_line l, account_move i - WHERE (l.move_id = i.id) - AND i.partner_id = %s - AND i.move_type = 'in_refund' - """, - (partner_id,), - ) - value -= self._cr.fetchone()[0] or 0.0 - res = expression[line["expr_operator"]](value, line["expr_value"]) - - if not res and (line["operator"] == "and"): - return False - elif res: - return True - return True + for partner_id in map(int, partners): + expression = { + "<": lambda x, y: x < y, + "=": lambda x, y: x == y, + ">": lambda x, y: x > y, + } + data = [] + for line in self: + self.env.cr.execute( + """ + SELECT * FROM ir_module_module WHERE name=%s AND state=%s + """, + ("account", "installed"), + ) + + if self.env.cr.fetchone(): + if line["expr_name"] == "sale": + self._cr.execute( + """SELECT SUM(l.price_unit * l.quantity) + FROM account_move_line l, account_move i + WHERE (l.move_id = i.id) AND + i.partner_id = %s AND + i.move_type = 'out_invoice' + """, + (partner_id,), + ) + value = self.env.cr.fetchone()[0] or 0.0 + self.env.cr.execute( + """SELECT SUM(l.price_unit * l.quantity) + FROM account_move_line l, account_move i + WHERE (l.move_id = i.id) + AND i.partner_id = %s + AND i.move_type = 'out_refund' + """, + (partner_id,), + ) + value -= self.env.cr.fetchone()[0] or 0.0 + elif line["expr_name"] == "purchase": + self.env.cr.execute( + """SELECT SUM(l.price_unit * l.quantity) + FROM account_move_line l, account_move i + WHERE (l.move_id = i.id) + AND i.partner_id = %s + AND i.move_type = 'in_invoice' + """, + (partner_id,), + ) + value = self.env.cr.fetchone()[0] or 0.0 + self.env.cr.execute( + """SELECT SUM(l.price_unit * l.quantity) + FROM account_move_line l, account_move i + WHERE (l.move_id = i.id) + AND i.partner_id = %s + AND i.move_type = 'in_refund' + """, + (partner_id,), + ) + value -= self._cr.fetchone()[0] or 0.0 + res = expression[line["expr_operator"]](value, line["expr_value"]) + + if not res and (line["operator"] == "and"): + data.append(False) + elif res: + data.append(True) + if not all(data): + to_remove_list.append(partner_id) + partnerd = list(partners) + for pid in to_remove_list: + partnerd.remove(pid) + + if self[0].segmentation_id.profiling_active: + to_remove_list = [] + for pid in partnerd: + self.env.cr.execute( + """ + SELECT DISTINCT(answer) FROM partner_question_rel + WHERE partner=%s""", + (pid,), + ) + answers_ids = [x[0] for x in self.env.cr.fetchall()] + + if not self[0].segmentation_id.test_prof(pid, answers_ids): + to_remove_list.append(pid) + for pid in to_remove_list: + partnerd.remove(pid) + for partner in self.env["res.partner"].browse(partnerd): + category_ids = partner.category_id.ids + if self[0].segmentation_id.categ_id.ids not in category_ids: + self._cr.execute( + """ + INSERT INTO res_partner_res_partner_category_rel + (category_id,partner_id) + VALUES (%s,%s) ON CONFLICT DO NOTHING""", + (self[0].segmentation_id.categ_id.id, partner.id), + ) + self.env["res.partner"].invalidate_cache(["category_id"], [partner.id]) diff --git a/crm_profiling/models/queue_job_batch.py b/crm_profiling/models/queue_job_batch.py new file mode 100644 index 00000000..e045b66f --- /dev/null +++ b/crm_profiling/models/queue_job_batch.py @@ -0,0 +1,9 @@ +from odoo import fields, models + + +class QueueJobBatch(models.Model): + _inherit = "queue.job.batch" + + segmentation_id = fields.Many2one( + comodel_name="crm.segmentation", inverse_name="job_batch_ids" + ) diff --git a/crm_profiling/views/crm_segmentation_view.xml b/crm_profiling/views/crm_segmentation_view.xml index 7e22bf76..c86a7b34 100644 --- a/crm_profiling/views/crm_segmentation_view.xml +++ b/crm_profiling/views/crm_segmentation_view.xml @@ -53,12 +53,14 @@