feat: allow users to submit category_tag with language prefix (#548)

Co-authored-by: Raphael Odini <[email protected]>

open_prices/prices/models.py

@@ -1,12 +1,17 @@
 import decimal
+import functools
 
 from django.core.validators import MinValueValidator, ValidationError
 from django.db import models
 from django.db.models import Avg, Count, F, Max, Min, signals
 from django.db.models.functions import Cast
 from django.dispatch import receiver
 from django.utils import timezone
-from openfoodfacts.taxonomy import get_taxonomy
+from openfoodfacts.taxonomy import (
+    create_taxonomy_mapping,
+    get_taxonomy,
+    map_to_canonical_id,
+)
 
 from open_prices.common import constants, utils
 from open_prices.locations import constants as location_constants
@@ -17,6 +22,12 @@
 from open_prices.proofs.models import Proof
 from open_prices.users.models import User
 
+# Taxonomy mapping generation takes ~200ms, so we cache it to avoid
+# recomputing it for each request.
+_cached_create_taxonomy_mapping = functools.lru_cache(maxsize=1)(
+    create_taxonomy_mapping
+)
+
 
 class PriceQuerySet(models.QuerySet):
     def exclude_discounted(self):
@@ -215,18 +226,36 @@ def clean(self, *args, **kwargs):
                     "price_per",
                     "Should not be set if `product_code` is filled",
                 )
-        # category_tag rules
-        # - if category_tag is set, then should be a valid taxonomy string
+        # Tag rules:
+        # - if category_tag is set, it should be language-prefixed
         # - if labels_tags is set, then all labels_tags should be valid taxonomy strings  # noqa
         # - if origins_tags is set, then all origins_tags should be valid taxonomy strings  # noqa
         elif self.category_tag:
             category_taxonomy = get_taxonomy("category")
-            if self.category_tag not in category_taxonomy:
+            # category_tag can be provided by the mobile app in any language,
+            # with language prefix (ex: `fr: Boissons`). We need to map it to
+            # the canonical id (ex: `en:beverages`) to store it in the
+            # database.
+            # The `map_to_canonical_id` function maps the value (ex:
+            # `fr: Boissons`) to the canonical id (ex: `en:beverages`).
+            # We use the cached version of this function to avoid
+            # creating it multiple times.
+            # If the entry does not exist in the taxonomy, category_tag will
+            # be set to the tag version of the value (ex: `fr:boissons`).
+            taxonomy_mapping = _cached_create_taxonomy_mapping(category_taxonomy)
+            try:
+                mapped_tags = map_to_canonical_id(taxonomy_mapping, [self.category_tag])
+            except ValueError as e:
+                # The value is not language-prefixed
                 validation_errors = utils.add_validation_error(
                     validation_errors,
                     "category_tag",
-                    f"Invalid category tag: category '{self.category_tag}' does not exist in the taxonomy",
+                    str(e),
                 )
+            else:
+                # Set the canonical id (or taggified version) as the
+                # category_tag
+                self.category_tag = mapped_tags[self.category_tag]
             if self.labels_tags:
                 if not isinstance(self.labels_tags, list):
                     validation_errors = utils.add_validation_error(

open_prices/prices/tests.py

@@ -108,11 +108,20 @@ def test_price_without_product_validation(self):
             price=3,
             price_per=price_constants.PRICE_PER_KILOGRAM,
         )
-        self.assertRaises(
-            ValidationError,
-            PriceFactory,
+        with self.assertRaises(ValidationError) as cm:
+            PriceFactory(
+                product_code=None,
+                category_tag="test",
+                price=3,
+                price_per=price_constants.PRICE_PER_KILOGRAM,
+            )
+        self.assertEqual(
+            cm.exception.messages[0],
+            "Invalid value: 'test', expected value to be in 'lang:tag' format",
+        )
+        PriceFactory(
             product_code=None,
-            category_tag="test",
+            category_tag="fr: Grenoble",  # valid (even if not in the taxonomy)
             price=3,
             price_per=price_constants.PRICE_PER_KILOGRAM,
         )
@@ -128,7 +137,7 @@ def test_price_without_product_validation(self):
             PriceFactory,
             product_code=None,
             category_tag="en:tomatoes",
-            labels_tags="en:organic",
+            labels_tags="en:organic",  # should be a list
             price=3,
             price_per=price_constants.PRICE_PER_KILOGRAM,
         )
@@ -137,7 +146,7 @@ def test_price_without_product_validation(self):
             PriceFactory,
             product_code=None,
             category_tag="en:tomatoes",
-            labels_tags=["en:organic", "test"],
+            labels_tags=["en:organic", "test"],  # not valid
             price=3,
             price_per=price_constants.PRICE_PER_KILOGRAM,
         )
@@ -155,7 +164,7 @@ def test_price_without_product_validation(self):
             product_code=None,
             category_tag="en:tomatoes",
             labels_tags=["en:organic"],
-            origins_tags="en:france",
+            origins_tags="en:france",  # should be a list
             price=3,
             price_per=price_constants.PRICE_PER_KILOGRAM,
         )
@@ -165,7 +174,7 @@ def test_price_without_product_validation(self):
             product_code=None,
             category_tag="en:tomatoes",
             labels_tags=["en:organic"],
-            origins_tags=["en:france", "test"],
+            origins_tags=["en:france", "test"],  # not valid
             price=3,
             price_per=price_constants.PRICE_PER_KILOGRAM,
         )
@@ -174,6 +183,20 @@ def test_price_without_product_validation(self):
             ValidationError, PriceFactory, product_code="", category_tag=""
         )
 
+    def test_price_category_validation(self):
+        for input_category, expected_category in [
+            ("en: Tomatoes", "en:tomatoes"),
+            ("fr: Pommes", "en:apples"),
+            ("fr: Soupe aux lentilles", "en:lentil-soups"),
+        ]:
+            price = PriceFactory(
+                product_code=None,
+                category_tag=input_category,
+                price=3,
+                price_per=price_constants.PRICE_PER_KILOGRAM,
+            )
+            self.assertEqual(price.category_tag, expected_category)
+
     def test_price_price_validation(self):
         for PRICE_OK in [5, 0]:
             PriceFactory(price=PRICE_OK)