Merge branch 'main' into crontask-packager-codes

.github/labeler.yml

@@ -689,7 +689,7 @@ Numbers:
   - any-glob-to-any-file: 'tests/unit/ingredients_nutriscore.t'
   - any-glob-to-any-file: 'tests/unit/ingredients_nutriscore.t'
   - any-glob-to-any-file: 'scripts/add_nutriscore_to_scanbot_csv.pl'
-  - any-glob-to-any-file: 'docs/introduction/get-the-nutriscore.md'
+  - any-glob-to-any-file: 'docs/api/api-tutorials/get-the-nutriscore.md'
   - any-glob-to-any-file: 'docs/​api/​ref/​schemas/​product_nutriscore.yaml'
 
 Nutrients:
@@ -1161,8 +1161,9 @@ OCR:
 Scanbot:
 - changed-files:
   - any-glob-to-any-file: 'scripts/scanbot.pl'
-
-
+  - any-glob-to-any-file: 'scripts/export_scans_to_query.pl'
+  - any-glob-to-any-file: 'scripts/add_nutriscore_to_scanbot_csv.pl'
+
 # We used to have a nice missions system with badges you'd earn by doing certain contribution tasks. It was disabled when we made Open Food Facts multilingual, and we never got to refactor it.
 # Tracking issue: https://github.com/openfoodfacts/openfoodfacts-server/issues/5516
 # https://openfoodfacts.github.io/openfoodfacts-server/dev/ref-perl-pod/ProductOpener/Missions.html

crowdin.yml

@@ -22,82 +22,6 @@ files:
         sr: sr_RS
         ku : kmr_TR
         kmr: kmr
-  - source: /po/openbeautyfacts/openbeautyfacts.pot
-    translation: /po/openbeautyfacts/%two_letters_code%.po
-    ignore: ["/po/openbeautyfacts/nl.po"]
-    languages_mapping:
-      two_letters_code:
-        en-GB: en_GB
-        pt-BR: pt_BR
-        pt-PT: pt_PT
-        en-AU: en_AU
-        en-US: en_US
-        zh-CN: zh_CN
-        zh-HK: zh_HK
-        nl-BE: nl_BE
-        nl: nl_NL
-        zh-TW: zh_TW
-        sr-CS: sr_CS
-        sr: sr_RS
-        ku : kmr_TR
-        kmr: kmr
-  - source: /po/openfoodfacts/openfoodfacts.pot
-    translation: /po/openfoodfacts/%two_letters_code%.po
-    ignore: ["/po/openfoodfacts/nl.po"]
-    languages_mapping:
-      two_letters_code:
-        zh-HK: zh_HK
-        zh-CN: zh_CN
-        en-AU: en_AU
-        en-GB: en_GB
-        en-US: en_US
-        nl-BE: nl_BE
-        nl: nl_NL
-        pt-BR: pt_BR
-        pt-PT: pt_PT
-        zh-TW: zh_TW
-        sr-CS: sr_CS
-        sr: sr_RS
-        ku : kmr_TR
-        kmr: kmr
-  - source: /po/openproductsfacts/openproductsfacts.pot
-    translation: /po/openproductsfacts/%two_letters_code%.po
-    ignore: ["/po/openproductsfacts/nl.po"]
-    languages_mapping:
-      two_letters_code:
-        zh-HK: zh_HK
-        zh-CN: zh_CN
-        en-AU: en_AU
-        en-GB: en_GB
-        en-US: en_US
-        nl-BE: nl_BE
-        nl: nl_NL
-        pt-BR: pt_BR
-        pt-PT: pt_PT
-        zh-TW: zh_TW
-        sr-CS: sr_CS
-        sr: sr_RS
-        ku : kmr_TR
-        kmr: kmr
-  - source: /po/openpetfoodfacts/openpetfoodfacts.pot
-    translation: /po/openpetfoodfacts/%two_letters_code%.po
-    ignore: ["/po/openpetfoodfacts/nl.po"]
-    languages_mapping:
-      two_letters_code:
-        zh-HK: zh_HK
-        zh-CN: zh_CN
-        en-AU: en_AU
-        en-GB: en_GB
-        en-US: en_US
-        pt-BR: pt_BR
-        pt-PT: pt_PT
-        nl-BE: nl_BE
-        nl: nl_NL
-        zh-TW: zh_TW
-        sr-CS: sr_CS
-        sr: sr_RS
-        ku : kmr_TR
-        kmr: kmr
   - source: /html/donate/en.html
     translation: /html/donate/%two_letters_code%.html
     languages_mapping:

docs/api/api-tutorials/get-the-nutriscore.md

@@ -0,0 +1,114 @@
+
+### Introduction
+-   If you can't get the information on a specific product, you can get your user to send photos and data, that will then be processed by Open Food Facts AI and contributors to get the computed result you want to show them.
+-   You can implement the complete flow below so that they get immediately the result with some effort on their side.
+-   That will ensure user satisfaction
+-   Please refer to the [product addition tutorial](https://openfoodfacts.github.io/openfoodfacts-server/reference/api-tutorials/adding-missing-products/) for the technical way to do the required operations (such as nutrition input), and to the high level workflow below for all the cases you have to handle.
+
+### Display Nutri-Score knowledge panels - All the logic below in 5 lines of code !
+- The Knowledge Panels are already implemented in the Dart package
+- They are simple to implement from the JSON API
+- They allow you to consume present and future knowledge from Open Food Facts
+
+### Using the official visual assets of the Nutri-Score
+
+Please use only the official assets to display the Nutri-Score. You can get v1 logos here: [NutriScore variants](https://drive.google.com/drive/u/1/folders/13SL2hgqYHSLMhYjMze9nYXV9GOdGMBgc)
+
+### Getting ready for Nutri-Score V2
+- Nutri-Score V2 has a new computation method, which now requires the ingredient list, a category, and of course the nutrition table
+- It also has a transition period new logo, to indicate you are using the new computation. It is not compulsory to use it, but it will save you from a lot of questions from your users ("Do you have the new formula ?")
+- The assets for the transition period logo are language dependant. We have already loaded the official assets. If your language is not supported, please get in touch instead of trying to translate the assets on your own. We'll make sure to find solutions.
+
+#### New transition assets for Nutri-Score V2
+- https://static.openfoodfacts.org/images/attributes/dist/nutriscore-a-new-en.svg
+- https://static.openfoodfacts.org/images/attributes/dist/nutriscore-b-new-en.svg
+- https://static.openfoodfacts.org/images/attributes/dist/nutriscore-c-new-en.svg
+- https://static.openfoodfacts.org/images/attributes/dist/nutriscore-d-new-en.svg
+- https://static.openfoodfacts.org/images/attributes/dist/nutriscore-e-new-en.svg
+
+#### Available languages
+- Dutch (nl), French (fr), German (de), Luxembourgish (lb), English (en)
+- Replace likeso `nutriscore-e-new-en.svg` > `nutriscore-e-new-fr.svg`
+
+- You can get the new assets in additional languages by contacting <a href="mailto:[email protected]">[email protected]</a>. We will provide unofficial courtesy translations.
+-  Make sure you use v2 assets when showing v2 values.
+
+### Manual version: Getting the Nutri-Score v1 value (we don't recommand the manual way anymore, especially with v2 around the corner)
+
+### Data completion flow
+
+Here are the different messages to use according to the state:
+
+#### Add a message if we have a category but no Nutri-Score
+
+<pre>if "en:categories-completed" in states_tags AND nutrition_grade=Null</pre>
+
+<pre>"We could not compute an Nutri-Score for this product. It might be that the category is an exception. If you believe this is an error, you can email [email protected]"</pre>
+
+-   List of exceptions: <https://www.santepubliquefrance.fr/content/download/150262/file/QR_scientifique_technique_150421.pdf>
+-   You can get states with [https://world.openfoodfacts.org/api/v0/product/3414280980209.json?fields=ecoscore_grade,states_tags ](https://world.openfoodfacts.org/api/v0/product/3414280980209.json?fields=ecoscore_grade,states_tags)
+
+#### Add a message if we have a category but no nutrition
+
+<pre>if "en:categories-completed" in states_tags  AND "en:nutrition-facts-to-be-completed" in states_tags</pre>pre>
+
+-   Prompt: "Add nutrition facts to compute the Nutri-Score"
+
+-   Add a one-click option to indicate no nutrition facts on the packaging
+  -   "This product doesn't have nutrition facts"
+
+#### Add a message if we have nutrition but no category
+
+<pre>if "en:categories-to-be-completed" in states_tags AND "en:nutrition-facts-completed" in states_tags</pre>
+
+-   Prompt: "Add a category to compute the Nutri-Score"
+
+#### Help the user add the category if it is missing
+
+-   You can use our Robotoff API to get your users to validate a prediction
+
+-   [Robotoff Questions](https://docs.google.com/document/d/1IoDy0toQrrqtWHvDYp2rEVw84Yq1J0x2pt-0RGTm7h0/edit)
+
+#### Add a message if we have no category and no nutrition
+
+<pre>if "en:categories-to-be-completed" in states_tags  AND "en:nutrition-facts-to-be-completed" in states_tags</pre>
+
+-   Prompt: "Add nutrition facts and a category to compute the Nutri-Score"
+
+#### Add a one-click option to indicate no nutrition facts on the packaging
+
+-   This product doesn't have nutrition facts
+
+#### Add a message if the nutrition image is missing
+
+<pre>if "en:nutrition-photo-to-be-selected" in states_tags OR "en:photos-to-be-uploaded" in states_tags</pre>
+
+#### Add a message if the nutrition image is obsolete using the image refresh API
+
+-   <https://github.com/openfoodfacts/api-documentation/issues/15>
+
+#### Add Nutri-Score disclaimers
+
+##### a message if fibers are missing
+<pre>
+msgctxt "nutrition_grade_fr_fiber_warning"
+msgid "Warning: the amount of fiber is not specified, their possible positive contribution to the grade could not be taken into account."
+</pre>
+
+##### a message if fruit/nuts are missing
+<pre>
+msgctxt "nutrition_grade_fr_no_fruits_vegetables_nuts_warning"
+msgid "Warning: the amount of fruits, vegetables and nuts is not specified, their possible positive contribution to the grade could not be taken into account."
+</pre>
+
+##### a message if fruits/nuts is an estimate from ingredients
+<pre>
+msgctxt "nutrition_grade_fr_fruits_vegetables_nuts_estimate_warning"
+msgid "Warning: the amount of fruits, vegetables and nuts is not specified on the label, it was estimated from the list of ingredients: %d%"
+</pre>
+
+##### a message if fruits/nuts is an estimate from category
+<pre>
+msgctxt "nutrition_grade_fr_fruits_vegetables_nuts_from_category_warning"
+msgid "Warning: the amount of fruits, vegetables and nuts is not specified on the label, it was estimated from the category (%s) of the product: %d%"
+</pre>

lib/ProductOpener/Config_off.pm

@@ -613,7 +613,6 @@ $options{categories_exempted_from_nutriscore} = [
 		en:spices
 		en:sugar-substitutes
 		en:vinegars
-		en:pet-food
 		en:non-food-products
 	)
 ];

lib/ProductOpener/DataQualityFood.pm

@@ -1504,6 +1504,17 @@ Checks related to the ingredients list and ingredients analysis.
 
 =cut
 
+# note: we need common ingredients words that exist only in 1 language
+my %ingredients_in_languages_regexps = (
+	"en" => qr/\b(sugar|salt|flour)\b/i,
+	"es" => qr/\b(azucar|agua|harina)\b/i,
+	"de" => qr/\b(zutaten|Zucker|Salz|Wasser|Mehl)\b/i,
+	"fr" => qr/\b(ingrédients|sucre|eau|sel|farine)\b/i,
+	"it" => qr/\b(ingredienti|zucchero|farina|acqua)\b/i,
+	"nl" => qr/\b(ingrediënten|suiker|zout|bloem)\b/i,
+	"pt" => qr/\b(açúcar|farinha|água)\b/i,
+);
+
 sub check_ingredients ($product_ref) {
 
 	# spell corrected additives
@@ -1513,19 +1524,22 @@ sub check_ingredients ($product_ref) {
 	}
 
 	# Multiple languages in ingredient lists
+	# Record the languages for which we have ingredients in the ingredients_text field
 
-	my $nb_languages = 0;
+	my %ingredients_in_languages = ();
 
 	if (defined $product_ref->{ingredients_text}) {
-		($product_ref->{ingredients_text} =~ /\b(ingrédients|sucre|eau|sel|farine)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(sugar|salt|flour|milk)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(ingrediënten|suiker|zout|bloem)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(azucar|agua|harina)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(zutaten|Zucker|Salz|Wasser|Mehl)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(açúcar|farinha|água)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(ingredienti|zucchero|farina|acqua)\b/i) and $nb_languages++;
+
+		foreach my $lc (keys %ingredients_in_languages_regexps) {
+
+			if ($product_ref->{ingredients_text} =~ $ingredients_in_languages_regexps{$lc}) {
+				$ingredients_in_languages{$lc} = 1;
+			}
+		}
 	}
 
+	my $nb_languages = scalar keys %ingredients_in_languages;
+
 	if ($nb_languages > 1) {
 		foreach my $max (5, 4, 3, 2, 1) {
 			if ($nb_languages > $max) {
@@ -1535,6 +1549,14 @@ sub check_ingredients ($product_ref) {
 		push @{$product_ref->{data_quality_warnings_tags}}, "en:ingredients-number-of-languages-$nb_languages";
 	}
 
+	# Create data quality warning for each language that is not the same as ingredients_lc
+	foreach my $lc (sort keys %ingredients_in_languages) {
+		if ($lc ne $product_ref->{ingredients_lc}) {
+			push @{$product_ref->{data_quality_warnings_tags}},
+				"en:ingredients-language-mismatch-" . $product_ref->{ingredients_lc} . "-contains-" . $lc;
+		}
+	}
+
 	if ((defined $product_ref->{ingredients_n}) and ($product_ref->{ingredients_n} > 0)) {
 
 		my $score = $product_ref->{unknown_ingredients_n} * 2 - $product_ref->{ingredients_n};

lib/ProductOpener/Products.pm

@@ -95,6 +95,7 @@ BEGIN {
 		&compute_completeness_and_missing_tags
 		&compute_product_history_and_completeness
 		&compute_languages
+		&review_product_type
 		&compute_changes_diff_text
 		&compute_data_sources
 		&compute_sort_keys
@@ -3000,6 +3001,59 @@ sub compute_languages ($product_ref) {
 	return;
 }
 
+=head2 review_product_type ( $product_ref )
+
+Reviews the product type based on the presence of specific tags in the categories field.
+Updates the product type if necessary.
+
+=head3 Arguments
+
+=head4 Product reference $product_ref
+
+A reference to a hash containing the product details.
+
+=cut
+
+sub review_product_type ($product_ref) {
+
+	my $error;
+
+	my $expected_type;
+	if (has_tag($product_ref, "categories", "en:open-beauty-facts")) {
+		$expected_type = "beauty";
+	}
+	elsif (has_tag($product_ref, "categories", "en:open-food-facts")) {
+		$expected_type = "food";
+	}
+	elsif (has_tag($product_ref, "categories", "en:open-pet-food-facts")) {
+		$expected_type = "petfood";
+	}
+	elsif (has_tag($product_ref, "categories", "en:open-products-facts")) {
+		$expected_type = "product";
+	}
+
+	if ($expected_type and ($product_ref->{product_type} ne $expected_type)) {
+		$error = change_product_type($product_ref, $expected_type);
+	}
+
+	if ($error) {
+		$log->error("review_product_type - error", {error => $error, product_ref => $product_ref});
+	}
+	else {
+		# We remove the tag en:incorrect-product-type and its children before the product is stored on the server of the new type
+		remove_tag($product_ref, "categories", "en:incorrect-product-type");
+		remove_tag($product_ref, "categories", "en:open-beauty-facts");
+		remove_tag($product_ref, "categories", "en:open-food-facts");
+		remove_tag($product_ref, "categories", "en:open-pet-food-facts");
+		remove_tag($product_ref, "categories", "en:open-products-facts");
+		remove_tag($product_ref, "categories", "en:non-food-products");
+		remove_tag($product_ref, "categories", "en:non-pet-food-products");
+		remove_tag($product_ref, "categories", "en:non-beauty-products");
+	}
+
+	return;
+}
+
 =head2 process_product_edit_rules ($product_ref)
 
 Process the edit_rules (see C<@edit_rules> in in Config file).
@@ -3723,6 +3777,11 @@ sub analyze_and_enrich_product_data ($product_ref, $response_ref) {
 
 	compute_languages($product_ref);    # need languages for allergens detection and cleaning ingredients
 
+	# change the product type of non-food categorized products (issue #11094)
+	if (has_tag($product_ref, "categories", "en:incorrect-product-type")) {
+		review_product_type($product_ref);
+	}
+
 	# Run special analysis, score calculations that it specific to the product type
 
 	if (($options{product_type} eq "food")) {