Merge branch 'main' into taxonomy/ita-feb

.github/labeler.yml

@@ -1161,8 +1161,9 @@ OCR:
 Scanbot:
 - changed-files:
   - any-glob-to-any-file: 'scripts/scanbot.pl'
-
-
+  - any-glob-to-any-file: 'scripts/export_scans_to_query.pl'
+  - any-glob-to-any-file: 'scripts/add_nutriscore_to_scanbot_csv.pl'
+
 # We used to have a nice missions system with badges you'd earn by doing certain contribution tasks. It was disabled when we made Open Food Facts multilingual, and we never got to refactor it.
 # Tracking issue: https://github.com/openfoodfacts/openfoodfacts-server/issues/5516
 # https://openfoodfacts.github.io/openfoodfacts-server/dev/ref-perl-pod/ProductOpener/Missions.html

crowdin.yml

@@ -22,82 +22,6 @@ files:
         sr: sr_RS
         ku : kmr_TR
         kmr: kmr
-  - source: /po/openbeautyfacts/openbeautyfacts.pot
-    translation: /po/openbeautyfacts/%two_letters_code%.po
-    ignore: ["/po/openbeautyfacts/nl.po"]
-    languages_mapping:
-      two_letters_code:
-        en-GB: en_GB
-        pt-BR: pt_BR
-        pt-PT: pt_PT
-        en-AU: en_AU
-        en-US: en_US
-        zh-CN: zh_CN
-        zh-HK: zh_HK
-        nl-BE: nl_BE
-        nl: nl_NL
-        zh-TW: zh_TW
-        sr-CS: sr_CS
-        sr: sr_RS
-        ku : kmr_TR
-        kmr: kmr
-  - source: /po/openfoodfacts/openfoodfacts.pot
-    translation: /po/openfoodfacts/%two_letters_code%.po
-    ignore: ["/po/openfoodfacts/nl.po"]
-    languages_mapping:
-      two_letters_code:
-        zh-HK: zh_HK
-        zh-CN: zh_CN
-        en-AU: en_AU
-        en-GB: en_GB
-        en-US: en_US
-        nl-BE: nl_BE
-        nl: nl_NL
-        pt-BR: pt_BR
-        pt-PT: pt_PT
-        zh-TW: zh_TW
-        sr-CS: sr_CS
-        sr: sr_RS
-        ku : kmr_TR
-        kmr: kmr
-  - source: /po/openproductsfacts/openproductsfacts.pot
-    translation: /po/openproductsfacts/%two_letters_code%.po
-    ignore: ["/po/openproductsfacts/nl.po"]
-    languages_mapping:
-      two_letters_code:
-        zh-HK: zh_HK
-        zh-CN: zh_CN
-        en-AU: en_AU
-        en-GB: en_GB
-        en-US: en_US
-        nl-BE: nl_BE
-        nl: nl_NL
-        pt-BR: pt_BR
-        pt-PT: pt_PT
-        zh-TW: zh_TW
-        sr-CS: sr_CS
-        sr: sr_RS
-        ku : kmr_TR
-        kmr: kmr
-  - source: /po/openpetfoodfacts/openpetfoodfacts.pot
-    translation: /po/openpetfoodfacts/%two_letters_code%.po
-    ignore: ["/po/openpetfoodfacts/nl.po"]
-    languages_mapping:
-      two_letters_code:
-        zh-HK: zh_HK
-        zh-CN: zh_CN
-        en-AU: en_AU
-        en-GB: en_GB
-        en-US: en_US
-        pt-BR: pt_BR
-        pt-PT: pt_PT
-        nl-BE: nl_BE
-        nl: nl_NL
-        zh-TW: zh_TW
-        sr-CS: sr_CS
-        sr: sr_RS
-        ku : kmr_TR
-        kmr: kmr
   - source: /html/donate/en.html
     translation: /html/donate/%two_letters_code%.html
     languages_mapping:

lib/ProductOpener/Config_off.pm

@@ -613,7 +613,6 @@ $options{categories_exempted_from_nutriscore} = [
 		en:spices
 		en:sugar-substitutes
 		en:vinegars
-		en:pet-food
 		en:non-food-products
 	)
 ];

lib/ProductOpener/DataQualityFood.pm

@@ -1504,6 +1504,17 @@ Checks related to the ingredients list and ingredients analysis.
 
 =cut
 
+# note: we need common ingredients words that exist only in 1 language
+my %ingredients_in_languages_regexps = (
+	"en" => qr/\b(sugar|salt|flour)\b/i,
+	"es" => qr/\b(azucar|agua|harina)\b/i,
+	"de" => qr/\b(zutaten|Zucker|Salz|Wasser|Mehl)\b/i,
+	"fr" => qr/\b(ingrédients|sucre|eau|sel|farine)\b/i,
+	"it" => qr/\b(ingredienti|zucchero|farina|acqua)\b/i,
+	"nl" => qr/\b(ingrediënten|suiker|zout|bloem)\b/i,
+	"pt" => qr/\b(açúcar|farinha|água)\b/i,
+);
+
 sub check_ingredients ($product_ref) {
 
 	# spell corrected additives
@@ -1513,19 +1524,22 @@ sub check_ingredients ($product_ref) {
 	}
 
 	# Multiple languages in ingredient lists
+	# Record the languages for which we have ingredients in the ingredients_text field
 
-	my $nb_languages = 0;
+	my %ingredients_in_languages = ();
 
 	if (defined $product_ref->{ingredients_text}) {
-		($product_ref->{ingredients_text} =~ /\b(ingrédients|sucre|eau|sel|farine)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(sugar|salt|flour|milk)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(ingrediënten|suiker|zout|bloem)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(azucar|agua|harina)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(zutaten|Zucker|Salz|Wasser|Mehl)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(açúcar|farinha|água)\b/i) and $nb_languages++;
-		($product_ref->{ingredients_text} =~ /\b(ingredienti|zucchero|farina|acqua)\b/i) and $nb_languages++;
+
+		foreach my $lc (keys %ingredients_in_languages_regexps) {
+
+			if ($product_ref->{ingredients_text} =~ $ingredients_in_languages_regexps{$lc}) {
+				$ingredients_in_languages{$lc} = 1;
+			}
+		}
 	}
 
+	my $nb_languages = scalar keys %ingredients_in_languages;
+
 	if ($nb_languages > 1) {
 		foreach my $max (5, 4, 3, 2, 1) {
 			if ($nb_languages > $max) {
@@ -1535,6 +1549,14 @@ sub check_ingredients ($product_ref) {
 		push @{$product_ref->{data_quality_warnings_tags}}, "en:ingredients-number-of-languages-$nb_languages";
 	}
 
+	# Create data quality warning for each language that is not the same as ingredients_lc
+	foreach my $lc (sort keys %ingredients_in_languages) {
+		if ($lc ne $product_ref->{ingredients_lc}) {
+			push @{$product_ref->{data_quality_warnings_tags}},
+				"en:ingredients-language-mismatch-" . $product_ref->{ingredients_lc} . "-contains-" . $lc;
+		}
+	}
+
 	if ((defined $product_ref->{ingredients_n}) and ($product_ref->{ingredients_n} > 0)) {
 
 		my $score = $product_ref->{unknown_ingredients_n} * 2 - $product_ref->{ingredients_n};

lib/ProductOpener/Products.pm

@@ -95,6 +95,7 @@ BEGIN {
 		&compute_completeness_and_missing_tags
 		&compute_product_history_and_completeness
 		&compute_languages
+		&review_product_type
 		&compute_changes_diff_text
 		&compute_data_sources
 		&compute_sort_keys
@@ -3000,6 +3001,59 @@ sub compute_languages ($product_ref) {
 	return;
 }
 
+=head2 review_product_type ( $product_ref )
+
+Reviews the product type based on the presence of specific tags in the categories field.
+Updates the product type if necessary.
+
+=head3 Arguments
+
+=head4 Product reference $product_ref
+
+A reference to a hash containing the product details.
+
+=cut
+
+sub review_product_type ($product_ref) {
+
+	my $error;
+
+	my $expected_type;
+	if (has_tag($product_ref, "categories", "en:open-beauty-facts")) {
+		$expected_type = "beauty";
+	}
+	elsif (has_tag($product_ref, "categories", "en:open-food-facts")) {
+		$expected_type = "food";
+	}
+	elsif (has_tag($product_ref, "categories", "en:open-pet-food-facts")) {
+		$expected_type = "petfood";
+	}
+	elsif (has_tag($product_ref, "categories", "en:open-products-facts")) {
+		$expected_type = "product";
+	}
+
+	if ($expected_type and ($product_ref->{product_type} ne $expected_type)) {
+		$error = change_product_type($product_ref, $expected_type);
+	}
+
+	if ($error) {
+		$log->error("review_product_type - error", {error => $error, product_ref => $product_ref});
+	}
+	else {
+		# We remove the tag en:incorrect-product-type and its children before the product is stored on the server of the new type
+		remove_tag($product_ref, "categories", "en:incorrect-product-type");
+		remove_tag($product_ref, "categories", "en:open-beauty-facts");
+		remove_tag($product_ref, "categories", "en:open-food-facts");
+		remove_tag($product_ref, "categories", "en:open-pet-food-facts");
+		remove_tag($product_ref, "categories", "en:open-products-facts");
+		remove_tag($product_ref, "categories", "en:non-food-products");
+		remove_tag($product_ref, "categories", "en:non-pet-food-products");
+		remove_tag($product_ref, "categories", "en:non-beauty-products");
+	}
+
+	return;
+}
+
 =head2 process_product_edit_rules ($product_ref)
 
 Process the edit_rules (see C<@edit_rules> in in Config file).
@@ -3723,6 +3777,11 @@ sub analyze_and_enrich_product_data ($product_ref, $response_ref) {
 
 	compute_languages($product_ref);    # need languages for allergens detection and cleaning ingredients
 
+	# change the product type of non-food categorized products (issue #11094)
+	if (has_tag($product_ref, "categories", "en:incorrect-product-type")) {
+		review_product_type($product_ref);
+	}
+
 	# Run special analysis, score calculations that it specific to the product type
 
 	if (($options{product_type} eq "food")) {