From 3c18781d6e4c4d8c95a26adf77a64ca8821f93e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Tue, 8 Oct 2024 15:45:44 +0200 Subject: [PATCH] fix: update paths for EAN8 and short barcodes (padding with zeroes) - DO NOT MERGE (#10472) I closed the very old PR https://github.com/openfoodfacts/openfoodfacts-server/pull/3915 and I am opening this new one to put it on the first page. Original issue: https://github.com/openfoodfacts/openfoodfacts-server/issues/3818 We will need to move the file structures on all flavors (OFF, OBF, OPF, OPFF) roughly at the same time, as otherwise we will have issues if some products are moved from one flavor to another. So the migration script is using "old" conventions so that it can run on the old obf / opf / opff code (without needing newer modules). The migration script is first used to assess the situation (how many products would be moved, how many products have conflicts etc.) on all flavors. PR in infrastructure repo to detail the migration: https://github.com/openfoodfacts/openfoodfacts-infrastructure/pull/363 --------- Co-authored-by: hangy Co-authored-by: Alex Garel Co-authored-by: OFF --- cgi/product_multilingual.pl | 2 +- conf/nginx/sites-available/obf | 11 +- conf/nginx/sites-available/off | 11 +- conf/nginx/sites-available/opf | 11 +- conf/nginx/sites-available/opff | 13 +- docs/api/how-to-download-images.md | 9 +- docs/api/ref-barcode-normalization.md | 34 + lib/ProductOpener/Display.pm | 11 +- lib/ProductOpener/Import.pm | 2 + lib/ProductOpener/Products.pm | 178 +++-- scripts/fix_non_normalized_codes.pl | 15 +- .../2024_10_move_ean8_products_to_new_path.pl | 663 ++++++++++++++++++ stop_words.txt | 2 + .../get-attribute-groups-fr.json | 2 +- .../get-attribute-groups.json | 2 +- .../get-auth-good-password.json | 4 +- .../get-existing-product.json | 29 +- .../get-fields-all-knowledge-panels.json | 29 +- .../api_v2_product_read/get-fields-all.json | 29 +- ...attribute-groups-all-knowledge-panels.json | 29 +- ...edge-panels_excluded-environment_card.json | 2 +- ...knowledge_panels_excluded-health_card.json | 2 +- ...included-health_card-environment_card.json | 2 +- .../api_v2_product_read/get-fields-raw.json | 29 +- .../get-knowledge-panels-fr.json | 2 +- .../get-knowledge-panels.json | 2 +- .../get-packagings-fr.json | 2 +- .../api_v2_product_read/get-packagings.json | 2 +- .../get-specific-fields.json | 2 +- .../get-existing-product-gs1-ai-data-str.json | 4 +- .../get-existing-product-gs1-caret.json | 4 +- .../get-existing-product-gs1-data-uri.json | 4 +- .../get-existing-product-gs1-fnc1.json | 4 +- .../get-existing-product-gs1-gs.json | 4 +- .../cors/get-api-v2.json | 2 +- .../data-quality.json | 2 +- .../no-data-quality.json | 2 +- .../expected_test_results/export/export.csv | 5 +- .../{25000044984.json => 0025000044984.json} | 2 +- .../{71464240608.json => 0071464240608.json} | 2 +- .../export/rows/04083637.json | 123 ++++ .../export_more_fields/export_more_fields.csv | 5 +- .../{25000044984.json => 0025000044984.json} | 2 +- .../{71464240608.json => 0071464240608.json} | 2 +- .../export_more_fields/rows/04083637.json | 149 ++++ .../product_read/get-existing-product.html | 37 +- .../product_read/get-unexisting-product.html | 2 +- ...ed-protected-product-api-v2-moderator.json | 1 + .../get-edited-protected-product-api-v2.json | 1 + ...-protected-product-web-form-moderator.json | 1 + ...get-edited-protected-product-web-form.json | 1 + ...get-edited-unprotected-product-api-v2.json | 1 + ...t-edited-unprotected-product-web-form.json | 1 + ...ch-fields-packagings-in-api-v3-format.json | 12 +- .../search_v1/search-fields-packagings.json | 12 +- .../search_v1/search-no-filter.json | 174 ++--- .../search_v1/search-specific-barcodes.json | 6 +- .../unknown_tags/country-france-exists.html | 4 +- .../unknown_tags/ingredient-apple-exists.html | 4 +- ...gredient-does-not-exist-but-not-empty.html | 4 +- .../web_html/fr-edit-product.html | 2 +- .../web_html/world-edit-product.html | 2 +- tests/integration/fix_non_normalized_codes.t | 38 +- tests/unit/products.t | 6 +- 64 files changed, 1430 insertions(+), 326 deletions(-) create mode 100644 docs/api/ref-barcode-normalization.md create mode 100755 scripts/migrations/2024_10_move_ean8_products_to_new_path.pl rename tests/integration/expected_test_results/export/rows/{25000044984.json => 0025000044984.json} (99%) rename tests/integration/expected_test_results/export/rows/{71464240608.json => 0071464240608.json} (99%) create mode 100644 tests/integration/expected_test_results/export/rows/04083637.json rename tests/integration/expected_test_results/export_more_fields/rows/{25000044984.json => 0025000044984.json} (99%) rename tests/integration/expected_test_results/export_more_fields/rows/{71464240608.json => 0071464240608.json} (99%) create mode 100644 tests/integration/expected_test_results/export_more_fields/rows/04083637.json diff --git a/cgi/product_multilingual.pl b/cgi/product_multilingual.pl index 2bdb12b61a6bc..9c8b9f80281db 100755 --- a/cgi/product_multilingual.pl +++ b/cgi/product_multilingual.pl @@ -338,7 +338,7 @@ ($product_ref) display_error_and_exit($request_ref, lang("no_owner_defined"), 200); } $product_id = product_id_for_owner($Owner_id, $code); - $product_ref = retrieve_product_or_deleted_product($product_id, $User{moderator}); + $product_ref = retrieve_product($product_id, $User{moderator}); if (not defined $product_ref) { display_error_and_exit($request_ref, sprintf(lang("no_product_for_barcode"), $code), 404); } diff --git a/conf/nginx/sites-available/obf b/conf/nginx/sites-available/obf index 8a9a302ff9931..a4f2c577b680b 100644 --- a/conf/nginx/sites-available/obf +++ b/conf/nginx/sites-available/obf @@ -46,6 +46,15 @@ server { index index.html index.htm index.nginx-debian.html; location ~ ^/images/products/ { + # 2024/10/03 - temporary redirects as we changed the path of images + # for barcodes that are 8 digits or less + rewrite ^/images/products/(....)/([^/]*)$ /images/products/000/000/000/$1/$2 break; + rewrite ^/images/products/(.)(....)/([^/]*)$ /images/products/000/000/00$1/$2/$3 break; + rewrite ^/images/products/(..)(....)/([^/]*)$ /images/products/000/000/0$1/$2/$3 break; + rewrite ^/images/products/(...)(....)/([^/]*)$ /images/products/000/000/$1/$2/$3 break; + rewrite ^/images/products/(.)(...)(....)/([^/]*)$ /images/products/000/00$1/$2/$3/$4 break; + rewrite ^/images/products/(..)(...)(....)/([^/]*)$ /images/products/000/0$1/$2/$3/$4 break; + include snippets/off.cors-headers.include; include snippets/expiry-headers.include; add_header Link "; rel='license'; title='CC-BY-SA 3.0'"; @@ -54,8 +63,6 @@ server { gunzip on; } - if ($http_referer ~* (jobothoniel.com) ) { return 403; } # blocked since 2021-07-13 - # the app requests /1.json to get the product count... # the commented code below is to serve a static copy # if there is a spike of installs diff --git a/conf/nginx/sites-available/off b/conf/nginx/sites-available/off index 0abce759dc10f..b2fcfb3565f5f 100644 --- a/conf/nginx/sites-available/off +++ b/conf/nginx/sites-available/off @@ -45,6 +45,15 @@ server { index index.html index.htm index.nginx-debian.html; location ~ ^/images/products/ { + # 2024/10/03 - temporary redirects as we changed the path of images + # for barcodes that are 8 digits or less + rewrite ^/images/products/(....)/([^/]*)$ /images/products/000/000/000/$1/$2 break; + rewrite ^/images/products/(.)(....)/([^/]*)$ /images/products/000/000/00$1/$2/$3 break; + rewrite ^/images/products/(..)(....)/([^/]*)$ /images/products/000/000/0$1/$2/$3 break; + rewrite ^/images/products/(...)(....)/([^/]*)$ /images/products/000/000/$1/$2/$3 break; + rewrite ^/images/products/(.)(...)(....)/([^/]*)$ /images/products/000/00$1/$2/$3/$4 break; + rewrite ^/images/products/(..)(...)(....)/([^/]*)$ /images/products/000/0$1/$2/$3/$4 break; + include snippets/off.cors-headers.include; include snippets/expiry-headers.include; add_header Link "; rel='license'; title='CC-BY-SA 3.0'"; @@ -76,8 +85,6 @@ server { return 302 https://openfoodfacts-ds.s3.eu-west-3.amazonaws.com/fr.openfoodfacts.org.products.csv.gz; } - if ($http_referer ~* (jobothoniel.com) ) { return 403; } # blocked since 2021-07-13 - # the app requests /1.json to get the product count... # the commented code below is to serve a static copy # if there is a spike of installs diff --git a/conf/nginx/sites-available/opf b/conf/nginx/sites-available/opf index 757c3d0213271..933ef77bce433 100644 --- a/conf/nginx/sites-available/opf +++ b/conf/nginx/sites-available/opf @@ -45,6 +45,15 @@ server { index index.html index.htm index.nginx-debian.html; location ~ ^/images/products/ { + # 2024/10/03 - temporary redirects as we changed the path of images + # for barcodes that are 8 digits or less + rewrite ^/images/products/(....)/([^/]*)$ /images/products/000/000/000/$1/$2 break; + rewrite ^/images/products/(.)(....)/([^/]*)$ /images/products/000/000/00$1/$2/$3 break; + rewrite ^/images/products/(..)(....)/([^/]*)$ /images/products/000/000/0$1/$2/$3 break; + rewrite ^/images/products/(...)(....)/([^/]*)$ /images/products/000/000/$1/$2/$3 break; + rewrite ^/images/products/(.)(...)(....)/([^/]*)$ /images/products/000/00$1/$2/$3/$4 break; + rewrite ^/images/products/(..)(...)(....)/([^/]*)$ /images/products/000/0$1/$2/$3/$4 break; + include snippets/off.cors-headers.include; include snippets/expiry-headers.include; add_header Link "; rel='license'; title='CC-BY-SA 3.0'"; @@ -53,8 +62,6 @@ server { gunzip on; } - if ($http_referer ~* (jobothoniel.com) ) { return 403; } # blocked since 2021-07-13 - # the app requests /1.json to get the product count... # the commented code below is to serve a static copy # if there is a spike of installs diff --git a/conf/nginx/sites-available/opff b/conf/nginx/sites-available/opff index 04d115b36cb1f..c892188d085c3 100644 --- a/conf/nginx/sites-available/opff +++ b/conf/nginx/sites-available/opff @@ -44,7 +44,16 @@ server { index index.html index.htm index.nginx-debian.html; - location ~ ^/images/petfood/ { + location ~ ^/images/products/ { + # 2024/10/03 - temporary redirects as we changed the path of images + # for barcodes that are 8 digits or less + rewrite ^/images/products/(....)/([^/]*)$ /images/products/000/000/000/$1/$2 break; + rewrite ^/images/products/(.)(....)/([^/]*)$ /images/products/000/000/00$1/$2/$3 break; + rewrite ^/images/products/(..)(....)/([^/]*)$ /images/products/000/000/0$1/$2/$3 break; + rewrite ^/images/products/(...)(....)/([^/]*)$ /images/products/000/000/$1/$2/$3 break; + rewrite ^/images/products/(.)(...)(....)/([^/]*)$ /images/products/000/00$1/$2/$3/$4 break; + rewrite ^/images/products/(..)(...)(....)/([^/]*)$ /images/products/000/0$1/$2/$3/$4 break; + include snippets/off.cors-headers.include; include snippets/expiry-headers.include; add_header Link "; rel='license'; title='CC-BY-SA 3.0'"; @@ -53,8 +62,6 @@ server { gunzip on; } - if ($http_referer ~* (jobothoniel.com) ) { return 403; } # blocked since 2021-07-13 - # the app requests /1.json to get the product count... # the commented code below is to serve a static copy # if there is a spike of installs diff --git a/docs/api/how-to-download-images.md b/docs/api/how-to-download-images.md index 001f4e8e85d99..b6847742cc085 100644 --- a/docs/api/how-to-download-images.md +++ b/docs/api/how-to-download-images.md @@ -42,14 +42,11 @@ In get you want to get an image which url is not directly present in product dat ### Computing single product image folder -Images of a product are stored in a single directory. The path of this -directory can be inferred easily from the product barcode. -There are two cases: +Images of a product are stored in a single directory. The path of this directory can be inferred easily from the product barcode: -1. If the product barcode is 8 digits long or shorter (ex: "22222222"), the directory path is -simply the barcode: `https://images.openfoodfacts.org/images/products/{barcode}`. +If the barcode is less than 13 digits long, it must be padded with leading 0s so that it has 13 digits. -2. Otherwise, split the first 9 digits of the barcode into 3 groups of 3 digits to get the first 3 folder names, and use the rest of the barcode as the last folder name^[split-regexp]. +Then split the first 9 digits of the barcode into 3 groups of 3 digits to get the first 3 folder names, and use the rest of the barcode as the last folder name^[split-regexp]. For example, barcode `3435660768163` is split into: `343/566/076/8163`, thus product images will be in `https://images.openfoodfacts.org/images/products/343/566/076/8163` ^[split-regexp]: The following regex can be used to split the barcode into subfolders: `/^(...)(...)(...)(.*)$/` diff --git a/docs/api/ref-barcode-normalization.md b/docs/api/ref-barcode-normalization.md new file mode 100644 index 0000000000000..a8221acdaa53a --- /dev/null +++ b/docs/api/ref-barcode-normalization.md @@ -0,0 +1,34 @@ +# Reference: Barcode Normalization + +This reference describes how barcodes are normalized in Open Food Facts. + +## The problem: barcodes may be prefixed by a varying number of 0s + +Different types of barcodes can be found on products. The most common are: + +* EAN-13 / GTIN-13: 13 digit barcode +* EAN-8: 8 digit barcode, short version of EAN-13 barcodes that have 5 leading 0s +* UPC-A / UPC-12: 12 digit barcode that were used mostly in the US and Canada. A leading 0 can be added to get the corresponding EAN-13. +* UPC-E: 7 digit barcode, short version of UPC-A +* EAN-14 / GTIN-14: used for non-consumer facing products (e.g. a case of individal products). a leading 0 can be added to EAN-13 to get the corresponding EAN-14. + +The same code could be printed on products with a different number of leading 0s. +Additionally, some barcode scanners may add or remove leading 0s. + +As the barcode is used as the key in Open Food Facts, we can end up with duplicate products that just differ by the number of leading 0s. + +## The solution: barcode normalization + +In Open Food Facts, we choose to fix the number of leading 0s in this way: + +All barcodes with 7 digits or less (after leading 0s are removed) are padded with leading 0s so that they have 8 digits. + +All barcodes with 9 to 12 digits are padded with leading 0s so that they have 13 digits. + +The "code" field in the product database, database dumps and exports is normalized in this way. + +### Normalization of barcodes in the API + +The Open Food Facts API automatically normalize the barcode passed in the "code" field for both READ and WRITE requests. + +So a request for the 12 digit barcode 034000470693 will return the product saved with "code" 0034000470693. \ No newline at end of file diff --git a/lib/ProductOpener/Display.pm b/lib/ProductOpener/Display.pm index 735b7727f01ed..2f7d0db778d6d 100644 --- a/lib/ProductOpener/Display.pm +++ b/lib/ProductOpener/Display.pm @@ -7897,12 +7897,13 @@ JS } # Old UPC-12 in url? Redirect to EAN-13 url - if ($request_code ne $code) { + # TODO - 2024/10/02 - Temporarily disabled so that we can migrate short barcodes with digits not equal to 8 or greater or equal to 13 + # Reenable after all products are migrated. + if (0 and ($request_code ne $code)) { $request_ref->{redirect} = $request_ref->{canon_url}; - $log->debug( - "302 redirecting user because request_code does not match code", - {redirect => $request_ref->{redirect}, lc => $lc, request_code => $code} - ) if $log->is_debug(); + $log->debug("302 redirecting user because request_code does not match code", + {redirect => $request_ref->{redirect}, lc => $lc, code => $code, request_code => $request_code}) + if $log->is_debug(); redirect_to_url($request_ref, 302, $request_ref->{redirect}); } diff --git a/lib/ProductOpener/Import.pm b/lib/ProductOpener/Import.pm index f0c2f58b2c763..18fad07dc3bf6 100644 --- a/lib/ProductOpener/Import.pm +++ b/lib/ProductOpener/Import.pm @@ -1492,6 +1492,7 @@ sub import_csv_file ($args_ref) { # read code my $code = $imported_product_ref->{code}; $code = normalize_code($code); + $imported_product_ref->{code} = $code; # In case we added or removed leading 0s my $modified = 0; @@ -2918,6 +2919,7 @@ sub update_export_status_for_csv_file ($args_ref) { my $code = $imported_product_ref->{code}; $code = normalize_code($code); + $imported_product_ref->{code} = $code; # In case we added or removed leading 0s my $product_id = product_id_for_owner($Owner_id, $code); $log->debug("update export status for product", {i => $i, code => $code, product_id => $product_id}) diff --git a/lib/ProductOpener/Products.pm b/lib/ProductOpener/Products.pm index 1eefc59a31789..b318d6576eaa8 100644 --- a/lib/ProductOpener/Products.pm +++ b/lib/ProductOpener/Products.pm @@ -73,6 +73,7 @@ BEGIN { &server_for_product_id &data_root_for_product_id &www_root_for_product_id + &split_code &product_path &product_path_from_id &product_id_from_path @@ -81,7 +82,6 @@ BEGIN { &get_owner_id &init_product &retrieve_product - &retrieve_product_or_deleted_product &retrieve_product_rev &store_product &product_name_brand @@ -289,11 +289,49 @@ sub normalize_code ($code) { if (defined $code) { ($code, my $gs1_ai_data_str) = &normalize_code_with_gs1_ai($code); + $code = normalize_code_zeroes($code); } return $code; } -=head2 normalize_code_with_gs1_ai() +=head2 normalize_code_zeroes($code) + +On disk, we store product files and images in directories named after the product code, and we add leading 0s to the paths. +So we need to normalize the number of leading 0s of product codes, so that we don't have 2 products for codes that differ only by leading 0s. + +This function normalizes the product code by: +- removing leading zeroes, +- adding leading zeroes to have at least 13 digits, +- removing leading zeroes for EAN8s to keep only 8 digits + +Note: this function adds leading 0s even if the GS1 code is not valid. + +=cut + +sub normalize_code_zeroes($code) { + + # Return the code as-is if it is not all digits + if ($code !~ /^\d+$/) { + return $code; + } + + # Remove leading zeroes + $code =~ s/^0+//; + + # Add leading zeroes to have at least 13 digits + if (length($code) < 13) { + $code = "0" x (13 - length($code)) . $code; + } + + # Remove leading zeroes for EAN8s to keep only 8 digits + if ((length($code) eq 13) and ($code =~ /^00000/)) { + $code = $'; + } + + return $code; +} + +=head2 normalize_code_with_gs1_ai($code) C this function normalizes the product code by: - running the given code through normalization method provided by GS1 to format a GS1 data string, or data URI to a GTIN, @@ -322,26 +360,6 @@ sub normalize_code_with_gs1_ai ($code) { # Keep only digits, remove spaces, dashes and everything else $code =~ s/\D//g; - - # Add a leading 0 to valid UPC-12 codes - # invalid 12 digit codes may be EAN-13s with a missing number - if ((length($code) eq 12) and ($ean_check->is_valid('0' . $code))) { - $code = '0' . $code; - } - - # Remove leading 0 for codes with 14 digits - if ((length($code) eq 14) and ($code =~ /^0/)) { - $code = $'; - } - - # Remove 5 or 6 leading 0s for EAN8 - # 00000080050100 (from Ferrero) - if ((length($code) eq 14) and ($code =~ /^000000/)) { - $code = $'; - } - if ((length($code) eq 13) and ($code =~ /^00000/)) { - $code = $'; - } } return ($code, $ai_data_str); } @@ -433,8 +451,42 @@ Example: 1234567890123 :- 123/456/789/0123 =cut +# We temporarily keep the old split_code() so that during the migration, +# we can check if the old path exists (in which case we use it) and if not, we use the new path. +sub old_split_code ($code) { + + # Require at least 4 digits (some stores use very short internal barcodes, they are likely to be conflicting) + if (not is_valid_code($code)) { + + $log->info("invalid code", {code => $code}) if $log->is_info(); + return "invalid"; + } + + # First splits into 3 sections of 3 numbers and the last section with the remaining numbers + my $path = $code; + # Remove leading 0s that were added to normalize the code + $code =~ s/^0+//; + if ($code =~ /^(.{3})(.{3})(.{3})(.*)$/) { + $path = "$1/$2/$3/$4"; + } + return $path; +} + sub split_code ($code) { + # TODO: remove old_split_code() once all products have been migrated to the new path + my $old_path = old_split_code($code); + # If the old path exists, the product has not been migrated yet, so we use the old path + # Note: this does not work on the pro platform, as we are missing the org-id component of the path. + if (-e "$BASE_DIRS{PRODUCTS}/$old_path/product.sto") { + $log->debug("old_split_code path exists, using old path", {code => $code, old_path => $old_path}) + if $log->is_debug(); + return $old_path; + } + else { + $log->debug("old_split_code path does not exist", {code => $code, old_path => $old_path}) if $log->is_debug(); + } + # Require at least 4 digits (some stores use very short internal barcodes, they are likely to be conflicting) if (not is_valid_code($code)) { @@ -442,7 +494,12 @@ sub split_code ($code) { return "invalid"; } - # First splits into 3 sections of 3 numbers and the ast section with the remaining numbers + # Pad code with 0s if it has less than 13 digits + if (length($code) < 13) { + $code = "0" x (13 - length($code)) . $code; + } + + # First splits into 3 sections of 3 numbers and the last section with the remaining numbers my $path = $code; if ($code =~ /^(.{3})(.{3})(.{3})(.*)$/) { $path = "$1/$2/$3/$4"; @@ -871,7 +928,7 @@ sub init_product ($userid, $orgid, $code, $countryid) { return $product_ref; } -sub retrieve_product ($product_id) { +sub retrieve_product ($product_id, $include_deleted = 0) { my $path = product_path_from_id($product_id); my $product_data_root = data_root_for_product_id($product_id); @@ -898,7 +955,7 @@ sub retrieve_product ($product_id) { if $log->is_debug(); } else { - if ($product_ref->{deleted}) { + if (($product_ref->{deleted}) and (not $include_deleted)) { $log->debug( "retrieve_product - deleted product", { @@ -934,36 +991,7 @@ sub retrieve_product ($product_id) { return $product_ref; } -sub retrieve_product_or_deleted_product ($product_id, $deleted_ok = 1) { - - my $path = product_path_from_id($product_id); - my $product_data_root = data_root_for_product_id($product_id); - - my $product_ref = retrieve("$product_data_root/products/$path/product.sto"); - - if ( (defined $product_ref) - and ($product_ref->{deleted}) - and (not $deleted_ok)) - { - return; - } - - if (defined $product_ref) { - # If the product is on another server, set the server field so that it will be saved in the other server if we save it - my $server = server_for_product_id($product_id); - if (defined $server) { - $product_ref->{server} = $server; - } - else { - # If the product was moved previously, it may have a server field, remove it - delete $product_ref->{server}; - } - } - - return $product_ref; -} - -sub retrieve_product_rev ($product_id, $rev) { +sub retrieve_product_rev ($product_id, $rev, $include_deleted = 0) { if ($rev !~ /^\d+$/) { return; @@ -976,7 +1004,7 @@ sub retrieve_product_rev ($product_id, $rev) { if (defined $product_ref) { - if ($product_ref->{deleted}) { + if (($product_ref->{deleted}) and (not $include_deleted)) { return; } @@ -1192,7 +1220,8 @@ sub store_product ($user_id, $product_ref, $comment) { if (defined $product_ref->{old_code}) { my $old_code = $product_ref->{old_code}; - my $old_path = product_path_from_id($old_code); + my $old_product_id = product_id_for_owner($Owner_id, $old_code); + my $old_path = product_path_from_id($old_product_id); if (defined $product_ref->{new_server}) { my $new_server = $product_ref->{new_server}; @@ -1221,6 +1250,25 @@ sub store_product ($user_id, $product_ref, $comment) { ensure_dir_created_or_die("$new_data_root/products/$prefix_path"); ensure_dir_created_or_die("$new_www_root/images/products/$prefix_path"); + # Check if we are updating the product in place: + # the code changed, but it is the same path + # this can happen if the path is already normalized, but the code is not + # in that case we just want to update the code, and remove the old one from MongoDB + # we don't need to move the directories + if ("$BASE_DIRS{PRODUCTS}/$old_path" eq "$new_data_root/products/$path") { + $log->debug("updating product code in place", {old_code => $old_code, code => $code}) if $log->is_debug(); + delete $product_ref->{old_code}; + # remove the old product from the previous collection + if ($delete_from_previous_products_collection) { + execute_query( + sub { + return $previous_products_collection->delete_one({"_id" => $product_ref->{_id}}); + } + ); + } + $product_ref->{_id} = $product_ref->{code} . ''; # treat id as string; + } + if ( (!-e "$new_data_root/products/$path") and (!-e "$new_www_root/images/products/$path")) { @@ -1463,6 +1511,9 @@ sub store_product ($user_id, $product_ref, $comment) { } # Publish information about update on Redis stream + $log->debug("push_to_redis_stream", + {code => $code, product_id => $product_id, action => $action, comment => $comment, diffs => $diffs}) + if $log->is_debug(); push_to_redis_stream($user_id, $product_ref, $action, $comment, $diffs); return 1; @@ -2162,12 +2213,13 @@ sub compute_product_history_and_completeness ($product_data_root, $current_produ # Read all previous versions to see which fields have been added or edited my @fields = ( - 'lang', 'product_name', - 'generic_name', @ProductOpener::Config::product_fields, - @ProductOpener::Config::product_other_fields, 'no_nutrition_data', - 'nutrition_data_per', 'nutrition_data_prepared_per', - 'serving_size', 'allergens', - 'traces', 'ingredients_text' + 'code', 'lang', + 'product_name', 'generic_name', + @ProductOpener::Config::product_fields, @ProductOpener::Config::product_other_fields, + 'no_nutrition_data', 'nutrition_data_per', + 'nutrition_data_prepared_per', 'serving_size', + 'allergens', 'traces', + 'ingredients_text' ); my %previous = (uploaded_images => {}, selected_images => {}, fields => {}, nutriments => {}); diff --git a/scripts/fix_non_normalized_codes.pl b/scripts/fix_non_normalized_codes.pl index 2a6fa19511227..0b15a386b0430 100755 --- a/scripts/fix_non_normalized_codes.pl +++ b/scripts/fix_non_normalized_codes.pl @@ -73,7 +73,8 @@ ($product_path, $product_id, $normalized_id) $product_ref->{code} = $normalized_id; $product_ref->{_id} = $normalized_id; # store updated (will move it) - store_product("fix-non-normalized-codes-script", $product_ref, "Normalize barcode"); + store_product("fix-non-normalized-codes-script", + $product_ref, "Normalize barcode from $product_id to $normalized_id"); return; } @@ -97,7 +98,15 @@ ($product_path, $dry_run, $out) my $path_from_old_id = product_path_from_id($product_id); my $is_duplicate = (-e "$BASE_DIRS{PRODUCTS}/$new_path"); my $is_invalid = $path_from_old_id eq "invalid"; - if ($is_duplicate || $is_invalid) { + # print "product_path: $product_path - new_path: $new_path - product_id: $product_id - normalized_id: $normalized_id - is_duplicate: $is_duplicate - is_invalid: $is_invalid - path_from_old_id: $path_from_old_id\n"; + # we could have different codes but the same path: EAN8 padded with 5 0s + # it happens in the test + if ($new_path eq $path_from_old_id) { + # we should change the code to the normalized code, store the new product, and remove the old code from MongoDB + move_product_to($product_path, $product_id, $normalized_id) unless $dry_run; + push(@actions, "Updated product in place: $product_id and $normalized_id have the same path $product_path"); + } + elsif ($is_duplicate || $is_invalid) { # this is probably older data than the normalized one, we will ditch it ! delete_product($product_path) unless $dry_run; my $msg = "Removed $product_id"; @@ -149,7 +158,7 @@ ($int_ids_ref, $dry_run) foreach my $int_id (@$int_ids_ref) { # load my $str_code = "$int_id"; - my $product_ref = retrieve_product_or_deleted_product($str_code); + my $product_ref = retrieve_product($str_code, "include_deleted"); if (defined $product_ref) { $product_ref->{_id} .= ''; $product_ref->{code} .= ''; diff --git a/scripts/migrations/2024_10_move_ean8_products_to_new_path.pl b/scripts/migrations/2024_10_move_ean8_products_to_new_path.pl new file mode 100755 index 0000000000000..79daa6b1f5e4f --- /dev/null +++ b/scripts/migrations/2024_10_move_ean8_products_to_new_path.pl @@ -0,0 +1,663 @@ +#!/usr/bin/perl -w + +# This file is part of Product Opener. +# +# Product Opener +# Copyright (C) 2011-2019 Association Open Food Facts +# Contact: contact@openfoodfacts.org +# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France +# +# Product Opener is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +use CGI::Carp qw(fatalsToBrowser); + +# use ProductOpener::PerlStandards; +# not available in old versions of ProductOpener running on obf, opf, opff + +use 5.24.0; +use strict; +use warnings; +use feature (qw/signatures :5.24/); +use utf8; + +use ProductOpener::Config qw/:all/; +use ProductOpener::Store qw/:all/; +use ProductOpener::Index qw/:all/; +use ProductOpener::Display qw/:all/; +use ProductOpener::Tags qw/:all/; +use ProductOpener::Users qw/:all/; +use ProductOpener::Images qw/:all/; +use ProductOpener::Lang qw/:all/; +use ProductOpener::Mail qw/:all/; +use ProductOpener::Products qw/:all/; +use ProductOpener::Food qw/:all/; +use ProductOpener::Ingredients qw/:all/; +use ProductOpener::Images qw/:all/; +use ProductOpener::Data qw/:all/; +use ProductOpener::Orgs qw/:all/; + +use CGI qw/:cgi :form escapeHTML/; +use URI::Escape::XS; +use Storable qw/dclone/; +use Encode; +use JSON::PP; + +use File::Copy (qw/move/); + +use Data::Dumper; + +sub normalize_code_zeroes($code) { + + # Remove leading zeroes + $code =~ s/^0+//; + + # Add leading zeroes to have at least 13 digits + if (length($code) < 13) { + $code = "0" x (13 - length($code)) . $code; + } + + # Remove leading zeroes for EAN8s + if ((length($code) eq 13) and ($code =~ /^00000/)) { + $code = $'; + } + + return $code; +} + +# This script includes a new_split_code() function so that we can run it even if Products.pm split_code() is old +# This is useful in particular for preparations before the move, e.g. to see which products would be affected + +sub is_valid_code ($code) { + # Return an empty string if $code is undef + return '' if !defined $code; + return $code =~ /^\d{4,24}$/; +} + +sub new_split_code ($code) { + + # Require at least 4 digits (some stores use very short internal barcodes, they are likely to be conflicting) + if (not is_valid_code($code)) { + return "invalid"; + } + + # Pad code with 0s if it has less than 13 digits + if (length($code) < 13) { + $code = "0" x (13 - length($code)) . $code; + } + + # First splits into 3 sections of 3 numbers and the last section with the remaining numbers + my $new_path = $code; + if ($code =~ /^(.{3})(.{3})(.{3})(.*)$/) { + $new_path = "$1/$2/$3/$4"; + } + return $new_path; +} + +sub new_product_path_from_id ($new_product_id) { + + my $new_product_id_without_server = $new_product_id; + $new_product_id_without_server =~ s/(.*)://; + + if ( (defined $server_options{private_products}) + and ($server_options{private_products}) + and ($new_product_id_without_server =~ /\//)) + { + return $` . "/" . new_split_code($'); + } + else { + return new_split_code($new_product_id_without_server); + } + +} + +sub ensure_dir_created_or_die ($new_path, $mode = oct(755)) { + # search base directory + my $prefix; + my $suffix; + my @base_dirs = ($data_root, $www_root); + foreach my $prefix_candidate (@base_dirs) { + if ($new_path =~ /^$prefix_candidate/) { + $prefix = $prefix_candidate; + $suffix = $'; + last; + } + } + if (!defined $prefix) { + die("Could not create $new_path, no corresponding base directory found in " . join(":", @base_dirs)); + return; + } + # ensure the rest of the path + foreach my $component (split(/\//, $suffix)) { + $prefix .= "/$component"; + (-e $prefix) or mkdir($prefix); + } + return (-e $new_path); +} + +sub product_dir_move($source, $target) { + # If the source does not contain directories, use move() to move it directly + # otherwise, create the target directory, go through all files in the source, and use move() to move them to the target, but do not move contained directories + # then remove the source directory + + # First check if source contains directories + my $contains_directories = 0; + opendir my $dh, $source or die "could not open $source directory: $!\n"; + foreach my $dir (sort readdir($dh)) { + chomp($dir); + next if $dir eq '.'; + next if $dir eq '..'; + next if $dir =~ /\.lock/; # lingering lock files + if (-d "$source/$dir") { + $contains_directories = 1; + last; + } + } + + # If source does not contain directories, use move() to move it directly + if (!$contains_directories) { + return move($source, $target); + } + + print STDERR "source $source contains directories, moving files instead of directory\n"; + + # Otherwise, create the target directory + ensure_dir_created_or_die($target); + + # Go through all files in the source + opendir my $dir_h, $source or die "could not open $source directory: $!\n"; + foreach my $file (sort readdir($dir_h)) { + chomp($file); + # Move files and symbolic links + if (!-d "$source/$file") { + move("$source/$file", "$target/$file") or die "could not move $source/$file to $target/$file: $!\n"; + } + } + + # Remove the source directory + rmdir($source); + + return 1; +} + +# Get a list of all products + +use Getopt::Long; + +my $move = 0; +my $move_conflicting_codes = 0; +my $product_paths_containing_other_products = 0; + +my $invalid = 0; +my $moved = 0; +my $not_moved = 0; +my $same_path = 0; +my $changed_code = 0; + +my $products_collection = get_products_collection(); +my $obsolete_products_collection = get_products_collection({obsolete => 1}); +my @orgids = (); + +GetOptions( + 'move' => \$move, + 'move-conflicting-codes' => \$move_conflicting_codes, + 'orgids=s' => \@orgids +); +@orgids = split(/,/, join(',', @orgids)); + +my $d = 0; + +open(my $log, ">>", "$data_root/logs/move_ean8_products_to_new_path.log"); +print $log "move_ean8_products_to_new_path.pl started at " . localtime() . "\n"; + +open(my $csv, ">>", "$data_root/logs/move_ean8_products_to_new_path.csv"); + +# Loop on organizations if we are on the producers platform +if ( (defined $server_options{private_products}) + and ($server_options{private_products})) +{ + if (scalar @orgids == 0) { + foreach my $file (sort glob("$data_root/products/*")) { + if ($file =~ /\/((user|org)-.*)$/) { + push @orgids, $1; + print "org: $1\n"; + } + } + } +} +else { + @orgids = (undef); +} + +foreach my $orgid (@orgids) { + + my $org_path = $orgid ? '/' . $orgid : ""; + + print STDERR "org_path: $org_path\n"; + + # Directories to store invalid codes and conflicting products + ensure_dir_created_or_die("$data_root/products$org_path/invalid-codes"); + ensure_dir_created_or_die("$data_root/products$org_path/conflicting-codes"); + ensure_dir_created_or_die("$www_root/images/products$org_path/invalid-codes"); + ensure_dir_created_or_die("$www_root/images/products$org_path/conflicting-codes"); + + my @products = (); + # Look for products with EAN8 codes directly in the product root + + my $dh; + + opendir $dh, "$data_root/products$org_path" + or die "could not open $data_root/products$org_path directory: $!\n"; + foreach my $dir (sort readdir($dh)) { + chomp($dir); + + print STDERR "org_path: $org_path - dir: $dir\n"; + + # Check it is a directory + next if not -d "$data_root/products$org_path/$dir"; + next if ($dir eq "invalid-codes"); + next if ($dir eq "conflicting-codes"); + + if ($dir =~ /^\d\d\d$/) { + + # We can have products with 9 to 12 digits that have a split path but that were not padded with 0s + # e.g. 000/001/112/22/ should be moved to 000/000/011/1222/ + opendir my $dh2, "$data_root/products$org_path/$dir" + or die "ERROR: could not open $data_root/products$org_path/$dir directory: $!\n"; + foreach my $dir2 (sort readdir($dh2)) { + chomp($dir2); + if ($dir2 =~ /^\d\d\d$/) { + opendir my $dh3, "$data_root/products$org_path/$dir/$dir2" + or die "ERROR: could not open $data_root/products$org_path/$dir/$dir2 directory: $!\n"; + foreach my $dir3 (sort readdir($dh3)) { + chomp($dir3); + if ($dir3 =~ /^\d\d\d$/) { + opendir my $dh4, "$data_root/products$org_path/$dir/$dir2/$dir3" + or die + "ERROR: could not open $data_root/products$org_path/$dir/$dir2/$dir3 directory: $!\n"; + my $level4_dirs = 0; + foreach my $dir4 (sort readdir($dh4)) { + chomp($dir4); + # We should have 4 digits or more (for codes with more than 13 digits) + if ($dir4 =~ /^\d+$/) { + if ($dir4 !~ /^\d\d\d\d/) { + + if (-e "$data_root/products$org_path/$dir/$dir2/$dir3/$dir4/product.sto") { + push @products, "$dir/$dir2/$dir3/$dir4"; + print STDERR + "nested dir with less than 13 digits: $dir/$dir2/$dir3/$dir4\n"; + print $log "nested dir with less than 13 digits: $dir/$dir2/$dir3/$dir4\n"; + $d++; + (($d % 1000) == 1) + and print STDERR "$d products - $dir/$dir2/$dir3/$dir4\n"; + } + } + $level4_dirs++; + } + + } + closedir $dh4; + + # Check if there is a product.sto file in the directory (happens when the barcode has 9 digits: the path is split, but there is no leftover) + if (-e "$data_root/products$org_path/$dir/$dir2/$dir3/product.sto") { + + print STDERR "nested dir with 9 digits: $dir/$dir2/$dir3\n"; + print $log "nested dir with 9 digits: $dir/$dir2/$dir3\n"; + + if ($level4_dirs == 0) { + push @products, "$dir/$dir2/$dir3"; + $d++; + print STDERR + "nested dir with 9 digits: $dir/$dir2/$dir3 --> does not have level 4 dirs, ok to move level 3 dir\n"; + print $log + "nested dir with 9 digits: $dir/$dir2/$dir3 --> does not have level 4 dirs, ok to move level 3 dir\n"; + } + else { + push @products, "$dir/$dir2/$dir3"; + $d++; + print STDERR + "nested dir 9 with digits: $dir/$dir2/$dir3 --> has $level4_dirs level 4 dirs, need to move files instead of dir\n"; + print $log + "nested dir with 9 digits: $dir/$dir2/$dir3 --> has $level4_dirs level 4 dirs, need to move files instead of dir\n"; + $product_paths_containing_other_products++; + } + $d++; + } + } + + } + closedir $dh3; + } + } + closedir $dh2; + + } + # Don't move dirs with 1 or 2 digits + elsif (($dir !~ /^\d\d?$/) and ($dir =~ /^\d+$/)) { + # Product directories at the root, with a different number than 3 digits + if (-e "$data_root/products$org_path/$dir/product.sto") { + push @products, $dir; + $d++; + (($d % 1000) == 1) and print STDERR "$d products - $dir\n"; + } + } + elsif ($dir !~ /^\.+$/) { + print STDERR "invalid code: $dir\n"; + print $log "invalid code: $dir\n"; + # Move the dir to $data_root/products$org_path/invalid-codes + if ($move) { + if (move("$data_root/products$org_path/$dir", "$data_root/products$org_path/invalid-codes/$dir")) { + print STDERR "moved invalid code $dir to $data_root/products$org_path/invalid-codes\n"; + print $log "moved invalid code $dir to $data_root/products$org_path/invalid-codes\n"; + } + else { + print STDERR "could not move invalid code $dir to $data_root/products$org_path/invalid-codes\n"; + print $log "could not move invalid code $dir to $data_root/products$org_path/invalid-codes\n"; + } + # Delete from mongodb + my $id = $org_path . "/" . $dir; + $id =~ s/^\///; + $products_collection->delete_one({_id => $id}); + $obsolete_products_collection->delete_one({_id => $id}); + + # Also move the image dir if it exists + if (-e "$www_root/images/products$org_path/$dir") { + if ( + move( + "$www_root/images/products$org_path/$dir", + "$www_root/images/products$org_path/invalid-codes/$dir" + ) + ) + { + print STDERR + "moved invalid code $dir images to $www_root/images/products$org_path/invalid-codes\n"; + print $log + "moved invalid code $dir images to $www_root/images/products$org_path/invalid-codes\n"; + } + else { + print STDERR + "could not move invalid code $dir images to $www_root/images/products$org_path/invalid-codes\n"; + print $log + "could not move invalid code $dir images to $www_root/images/products$org_path/invalid-codes\n"; + } + } + } + } + } + closedir $dh; + + my $count = scalar @products; + + print STDERR "$count products to update for org_path: $org_path\n"; + + foreach my $old_path (@products) { + + my $code = $old_path; + + # remove / if any + $code =~ s/\///g; + + my $new_code = normalize_code_zeroes($code); + + my $old_product_id = product_id_for_owner($orgid, $code); + my $new_product_id = product_id_for_owner($orgid, $new_code); + + my $old_path = $org_path . '/' . $old_path; + $old_path =~ s/^\///; + my $new_path = new_product_path_from_id($new_product_id); + + my $product_ref = retrieve_product($old_product_id, "include_deleted"); + + my $deleted = $product_ref->{deleted} ? "deleted" : ""; + my $obsolete = $product_ref->{obsolete} ? "obsolete" : ""; + + print $csv "$code\t$new_code\t$old_path\t$new_path\t$obsolete\t$deleted\n"; + + if ($new_path eq "invalid") { + $invalid++; + print STDERR "invalid path for code $code (old path: $old_path)\n"; + print $log "invalid path for code $code (old path: $old_path)\n"; + } + elsif ($new_path ne $old_path) { + + # If the new path already exists, we do not know if the conflicting product is better (more complete / more fresh) than the old product + # We check the size of the product.sto file to determine which product seems more complete + # EXCEPT if the old product is deleted, in which case we will delete the old path instead + + if ( (-e "$data_root/products/$new_path") + and (not $deleted) + and ($move) + and ($move_conflicting_codes) + and (-s "$data_root/products/$old_path/product.sto" > -s "$data_root/products/$new_path/product.sto")) + { + my $old_size = -s "$data_root/products/$old_path/product.sto"; + my $new_size = -s "$data_root/products/$new_path/product.sto"; + print STDERR + "$code - $obsolete - $deleted - new $new_path (new size: $new_size) already exists, keeping old (old size: $old_size) moving new $new_path to conflicting-codes/$new_code\n"; + print $log + "$code - $obsolete - $deleted - new $new_path (new size: $new_size) already exists, keeping old (old size: $old_size) moving new $new_path to conflicting-codes/$new_code\n"; + + if ($move) { + if ( + move( + "$data_root/products/$new_path", + "$data_root/products$org_path/conflicting-codes/$new_code" + ) + ) + { + print STDERR "moved $new_path to $data_root/products$org_path/conflicting-codes/$new_code\n"; + print $log "moved $new_path to $data_root/products$org_path/conflicting-codes/$new_code\n"; + } + else { + print STDERR + "could not move $new_path to $data_root/products$org_path/conflicting-codes/$new_code\n"; + print $log + "could not move $new_path to $data_root/products$org_path/conflicting-codes/$new_code\n"; + } + # Also move the image dir if it exists + if (-e "$www_root/images/products/$new_path") { + if ( + move( + "$www_root/images/products/$new_path", + "$www_root/images/products$org_path/conflicting-codes/$new_code" + ) + ) + { + print STDERR + "moved $new_path images to $www_root/images/products$org_path/conflicting-codes/$new_code\n"; + print $log + "moved $new_path images to $www_root/images/products$org_path/conflicting-codes/$new_code\n"; + } + else { + print STDERR + "could not move $new_path images to $www_root/images/products$org_path/conflicting-codes/$new_code\n"; + print $log + "could not move $new_path images to $www_root/images/products$org_path/conflicting-codes/$new_code\n"; + } + } + } + } + + if ( (!-e "$data_root/products/$new_path") + and (!-e "$www_root/images/products/$new_path")) + { + print STDERR "$code - $obsolete - $deleted - new $new_path does not exist, moving $old_path\n"; + print $log "$code - $obsolete - $deleted - new $new_path does not exist, moving $old_path\n"; + $moved++; + + if ($move) { + + my $prefix_path = $new_path; + $prefix_path =~ s/\/[^\/]+$//; # remove the last subdir: we'll move it + ensure_dir_created_or_die("$data_root/products/$prefix_path"); + ensure_dir_created_or_die("$www_root/images/products/$prefix_path"); + + if ( (!-e "$data_root/products/$new_path") + and (!-e "$www_root/images/products/$new_path")) + { + # File::Copy move() is intended to move files, not + # directories. It does work on directories if the + # source and target are on the same file system + # (in which case the directory is just renamed), + # but fails otherwise. + # An alternative is to use File::Copy::Recursive + # but then it will do a copy even if it is the same + # file system... + # Another option is to call the system mv command. + + # In this script, we want to avoid creating copies as we are using zfs, so we use File::Copy move() + + print STDERR ( + "moving product data $data_root/products/$old_path to $data_root/products/$new_path\n"); + print $log ( + "moving product data $data_root/products/$old_path to $data_root/products/$new_path\n"); + + if (not product_dir_move("$data_root/products/$old_path", "$data_root/products/$new_path")) { + print STDERR ( + "ERROR: could not move product data from $data_root/products/$old_path to $data_root/products/$new_path : $!\n" + ); + print $log ( + "ERROR: could not move product data from $data_root/products/$old_path to $data_root/products/$new_path : $!\n" + ); + $moved--; + $not_moved++; + } + elsif (-e "$www_root/images/products/$old_path") { + + print STDERR ( + "moving product images $www_root/images/products/$old_path to $www_root/images/products/$new_path\n" + ); + + print $log ( + "moving product images $www_root/images/products/$old_path to $www_root/images/products/$new_path\n" + ); + + if ( + not product_dir_move( + "$www_root/images/products/$old_path", + "$www_root/images/products/$new_path" + ) + ) + { + print STDERR ( + "ERROR: could not move product images from $www_root/images/products/$old_path to $www_root/images/products/$new_path : $!\n" + ); + print $log ( + "ERROR: could not move product images from $www_root/images/products/$old_path to $www_root/images/products/$new_path : $!\n" + ); + } + + # If the code changed, need to update the product .sto file and to remove the old code from MongoDB and to add the new code in MongoDB + if ($new_code ne $code) { + + my $product_ref = retrieve_product($new_product_id, "include_deleted"); + $product_ref->{code} = $new_code . ''; + $product_ref->{id} = $product_ref->{code} . ''; # treat id as string; + $product_ref->{_id} = $new_product_id . ''; # treat id as string; + # Delete the old code from MongoDB collections + $products_collection->delete_one({_id => $old_product_id}); + $obsolete_products_collection->delete_one({_id => $old_product_id}); + # If the product is not deleted, store_product will add the new code to MongoDB + store_product("fix-code-bot", $product_ref, "changed code from $code to $new_code"); + print STDERR "updated code from $code to $new_code in .sto file and MongoDB\n"; + print $log "updated code from $code to $new_code in .sto file and MongoDB\n"; + } + } + + } + #exit; + #($moved % 10 == 0) and exit; + } + + if ($new_code ne $code) { + $changed_code++; + print STDERR "changed code from $code to $new_code\n"; + print $log "changed code from $code to $new_code\n"; + } + } + else { + + if ($move_conflicting_codes) { + # Move product and images to conflicting-codes/$code + print STDERR + "new path $new_path exists, moving $old_path to $data_root/products$org_path/conflicting-codes/$code\n"; + print $log + "new path $new_path exists, moving $old_path to $data_root/products$org_path/conflicting-codes/$code\n"; + + $moved++; + + if ( + not product_dir_move( + "$data_root/products/$old_path", + "$data_root/products$org_path/conflicting-codes/$code" + ) + ) + { + print STDERR ( + "ERROR: could not move product data from $data_root/products/$old_path to $data_root/products$org_path/conflicting-codes/$code : $!\n" + ); + print $log ( + "ERROR: could not move product data from $data_root/products/$old_path to $data_root/products$org_path/conflicting-codes/$code : $!\n" + ); + $moved--; + $not_moved++; + } + elsif (-e "$www_root/images/products/$old_path") { + if ( + not product_dir_move( + "$www_root/images/products/$old_path", + "$www_root/images/products$org_path/conflicting-codes/$code" + ) + ) + { + print STDERR ( + "ERROR: could not move product images from $www_root/images/products/$old_path to $www_root/images/products$org_path/conflicting-codes/$code : $!\n" + ); + print $log ( + "ERROR: could not move product images from $www_root/images/products/$old_path to $www_root/images/products$org_path/conflicting-codes/$code : $!\n" + ); + } + } + + # Delete $code from mongodb collections + $products_collection->delete_one({_id => $old_product_id}); + $obsolete_products_collection->delete_one({_id => $old_product_id}); + + } + else { + print STDERR "new path exists, not moving $old_path to $new_path\n"; + print $log "new path exists, not moving $old_path to $new_path\n"; + $not_moved++; + } + } + + } + else { + print STDERR "new $new_path is the same as old $old_path\n"; + print $log "new $new_path is the same as old $old_path\n"; + $same_path++; + } + } + print STDERR "$count products at the root or not split into a 4 component path for org_path $org_path\n"; + +} + +print STDERR "$product_paths_containing_other_products products paths containing other products\n"; +print STDERR "invalid code: $invalid\n"; +print STDERR "moved: $moved\n"; +print STDERR "not moved: $not_moved\n"; +print STDERR "same path: $same_path\n"; +print STDERR "changed code: $changed_code\n"; + +exit(0); + diff --git a/stop_words.txt b/stop_words.txt index 08ca8af7ed42c..d43e00fdd4d50 100644 --- a/stop_words.txt +++ b/stop_words.txt @@ -69,6 +69,8 @@ dont dropdown du ecoscore +EAN +EANs eg emb Envol diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-attribute-groups-fr.json b/tests/integration/expected_test_results/api_v2_product_read/get-attribute-groups-fr.json index bb1199fb25f27..e376eff690baf 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-attribute-groups-fr.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-attribute-groups-fr.json @@ -1,5 +1,5 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { "attribute_groups" : [ { diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-attribute-groups.json b/tests/integration/expected_test_results/api_v2_product_read/get-attribute-groups.json index 23a15f7cf392e..75c1e19ede56e 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-attribute-groups.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-attribute-groups.json @@ -1,5 +1,5 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { "attribute_groups" : [ { diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-auth-good-password.json b/tests/integration/expected_test_results/api_v2_product_read/get-auth-good-password.json index 47cb4242de9b2..6d9ef9e8e9614 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-auth-good-password.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-auth-good-password.json @@ -1,7 +1,7 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { - "code" : "200000000034", + "code" : "0200000000034", "product_name" : "Some product" }, "status" : 1, diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-existing-product.json b/tests/integration/expected_test_results/api_v2_product_read/get-existing-product.json index 70e2b1da9745f..91c4d1f0bb073 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-existing-product.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-existing-product.json @@ -1,7 +1,7 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { - "_id" : "200000000034", + "_id" : "0200000000034", "_keywords" : [ "cookie", "organic", @@ -57,18 +57,19 @@ "en:biscuits" ], "checkers_tags" : [], - "code" : "200000000034", + "code" : "0200000000034", "codes_tags" : [ - "code-12", - "200000000xxx", - "20000000xxxx", - "2000000xxxxx", - "200000xxxxxx", - "20000xxxxxxx", - "2000xxxxxxxx", - "200xxxxxxxxx", - "20xxxxxxxxxx", - "2xxxxxxxxxxx" + "code-13", + "0200000000xxx", + "020000000xxxx", + "02000000xxxxx", + "0200000xxxxxx", + "020000xxxxxxx", + "02000xxxxxxxx", + "0200xxxxxxxxx", + "020xxxxxxxxxx", + "02xxxxxxxxxxx", + "0xxxxxxxxxxxx" ], "complete" : 0, "completeness" : 0.5, @@ -584,7 +585,7 @@ }, "generic_name" : "Tester", "generic_name_en" : "Tester", - "id" : "200000000034", + "id" : "0200000000034", "informers_tags" : [ "tests" ], diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-fields-all-knowledge-panels.json b/tests/integration/expected_test_results/api_v2_product_read/get-fields-all-knowledge-panels.json index e6cbf11c6d423..9134226de65f1 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-fields-all-knowledge-panels.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-fields-all-knowledge-panels.json @@ -1,7 +1,7 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { - "_id" : "200000000034", + "_id" : "0200000000034", "_keywords" : [ "cookie", "organic", @@ -57,18 +57,19 @@ "en:biscuits" ], "checkers_tags" : [], - "code" : "200000000034", + "code" : "0200000000034", "codes_tags" : [ - "code-12", - "200000000xxx", - "20000000xxxx", - "2000000xxxxx", - "200000xxxxxx", - "20000xxxxxxx", - "2000xxxxxxxx", - "200xxxxxxxxx", - "20xxxxxxxxxx", - "2xxxxxxxxxxx" + "code-13", + "0200000000xxx", + "020000000xxxx", + "02000000xxxxx", + "0200000xxxxxx", + "020000xxxxxxx", + "02000xxxxxxxx", + "0200xxxxxxxxx", + "020xxxxxxxxxx", + "02xxxxxxxxxxx", + "0xxxxxxxxxxxx" ], "complete" : 0, "completeness" : 0.5, @@ -584,7 +585,7 @@ }, "generic_name" : "Tester", "generic_name_en" : "Tester", - "id" : "200000000034", + "id" : "0200000000034", "informers_tags" : [ "tests" ], diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-fields-all.json b/tests/integration/expected_test_results/api_v2_product_read/get-fields-all.json index 0d3e50bd11d3e..52d575c8d224c 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-fields-all.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-fields-all.json @@ -1,7 +1,7 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { - "_id" : "200000000034", + "_id" : "0200000000034", "_keywords" : [ "cookie", "organic", @@ -57,18 +57,19 @@ "en:biscuits" ], "checkers_tags" : [], - "code" : "200000000034", + "code" : "0200000000034", "codes_tags" : [ - "code-12", - "200000000xxx", - "20000000xxxx", - "2000000xxxxx", - "200000xxxxxx", - "20000xxxxxxx", - "2000xxxxxxxx", - "200xxxxxxxxx", - "20xxxxxxxxxx", - "2xxxxxxxxxxx" + "code-13", + "0200000000xxx", + "020000000xxxx", + "02000000xxxxx", + "0200000xxxxxx", + "020000xxxxxxx", + "02000xxxxxxxx", + "0200xxxxxxxxx", + "020xxxxxxxxxx", + "02xxxxxxxxxxx", + "0xxxxxxxxxxxx" ], "complete" : 0, "completeness" : 0.5, @@ -584,7 +585,7 @@ }, "generic_name" : "Tester", "generic_name_en" : "Tester", - "id" : "200000000034", + "id" : "0200000000034", "informers_tags" : [ "tests" ], diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-fields-attribute-groups-all-knowledge-panels.json b/tests/integration/expected_test_results/api_v2_product_read/get-fields-attribute-groups-all-knowledge-panels.json index e4c525b64071b..03a19d9617b5b 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-fields-attribute-groups-all-knowledge-panels.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-fields-attribute-groups-all-knowledge-panels.json @@ -1,7 +1,7 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { - "_id" : "200000000034", + "_id" : "0200000000034", "_keywords" : [ "cookie", "organic", @@ -713,18 +713,19 @@ "en:biscuits" ], "checkers_tags" : [], - "code" : "200000000034", + "code" : "0200000000034", "codes_tags" : [ - "code-12", - "200000000xxx", - "20000000xxxx", - "2000000xxxxx", - "200000xxxxxx", - "20000xxxxxxx", - "2000xxxxxxxx", - "200xxxxxxxxx", - "20xxxxxxxxxx", - "2xxxxxxxxxxx" + "code-13", + "0200000000xxx", + "020000000xxxx", + "02000000xxxxx", + "0200000xxxxxx", + "020000xxxxxxx", + "02000xxxxxxxx", + "0200xxxxxxxxx", + "020xxxxxxxxxx", + "02xxxxxxxxxxx", + "0xxxxxxxxxxxx" ], "complete" : 0, "completeness" : 0.5, @@ -1240,7 +1241,7 @@ }, "generic_name" : "Tester", "generic_name_en" : "Tester", - "id" : "200000000034", + "id" : "0200000000034", "informers_tags" : [ "tests" ], diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_excluded-environment_card.json b/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_excluded-environment_card.json index 85106b937310e..63c2ffeb8ad03 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_excluded-environment_card.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_excluded-environment_card.json @@ -1,5 +1,5 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { "knowledge_panels" : { "health_card" : { diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_included-health_card-environment_card-knowledge_panels_excluded-health_card.json b/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_included-health_card-environment_card-knowledge_panels_excluded-health_card.json index ed4d76c416a43..708b708b2d5de 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_included-health_card-environment_card-knowledge_panels_excluded-health_card.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_included-health_card-environment_card-knowledge_panels_excluded-health_card.json @@ -1,5 +1,5 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { "knowledge_panels" : { "carbon_footprint" : { diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_included-health_card-environment_card.json b/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_included-health_card-environment_card.json index 61c6a778f681a..35752e10e9821 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_included-health_card-environment_card.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-fields-knowledge-panels-knowledge-panels_included-health_card-environment_card.json @@ -1,5 +1,5 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { "knowledge_panels" : { "carbon_footprint" : { diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-fields-raw.json b/tests/integration/expected_test_results/api_v2_product_read/get-fields-raw.json index b02e86d12071d..4141e76663a04 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-fields-raw.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-fields-raw.json @@ -1,7 +1,7 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { - "_id" : "200000000034", + "_id" : "0200000000034", "_keywords" : [ "cookie", "organic", @@ -57,18 +57,19 @@ "en:biscuits" ], "checkers_tags" : [], - "code" : "200000000034", + "code" : "0200000000034", "codes_tags" : [ - "code-12", - "200000000xxx", - "20000000xxxx", - "2000000xxxxx", - "200000xxxxxx", - "20000xxxxxxx", - "2000xxxxxxxx", - "200xxxxxxxxx", - "20xxxxxxxxxx", - "2xxxxxxxxxxx" + "code-13", + "0200000000xxx", + "020000000xxxx", + "02000000xxxxx", + "0200000xxxxxx", + "020000xxxxxxx", + "02000xxxxxxxx", + "0200xxxxxxxxx", + "020xxxxxxxxxx", + "02xxxxxxxxxxx", + "0xxxxxxxxxxxx" ], "complete" : 0, "completeness" : 0.5, @@ -579,7 +580,7 @@ }, "generic_name" : "Tester", "generic_name_en" : "Tester", - "id" : "200000000034", + "id" : "0200000000034", "informers_tags" : [ "tests" ], diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-knowledge-panels-fr.json b/tests/integration/expected_test_results/api_v2_product_read/get-knowledge-panels-fr.json index 45860d65908c9..91a48114dc672 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-knowledge-panels-fr.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-knowledge-panels-fr.json @@ -1,5 +1,5 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { "knowledge_panels" : { "carbon_footprint" : { diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-knowledge-panels.json b/tests/integration/expected_test_results/api_v2_product_read/get-knowledge-panels.json index 83340712e8621..631730c89fe5c 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-knowledge-panels.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-knowledge-panels.json @@ -1,5 +1,5 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { "knowledge_panels" : { "carbon_footprint" : { diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-packagings-fr.json b/tests/integration/expected_test_results/api_v2_product_read/get-packagings-fr.json index 709f0ee58e6dc..a627245d9dae7 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-packagings-fr.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-packagings-fr.json @@ -1,5 +1,5 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { "packagings" : [ { diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-packagings.json b/tests/integration/expected_test_results/api_v2_product_read/get-packagings.json index 709f0ee58e6dc..a627245d9dae7 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-packagings.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-packagings.json @@ -1,5 +1,5 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { "packagings" : [ { diff --git a/tests/integration/expected_test_results/api_v2_product_read/get-specific-fields.json b/tests/integration/expected_test_results/api_v2_product_read/get-specific-fields.json index 462c0f7e4d2b3..f29efcfb9713c 100644 --- a/tests/integration/expected_test_results/api_v2_product_read/get-specific-fields.json +++ b/tests/integration/expected_test_results/api_v2_product_read/get-specific-fields.json @@ -1,5 +1,5 @@ { - "code" : "200000000034", + "code" : "0200000000034", "product" : { "categories_tags" : [ "en:snacks", diff --git a/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-ai-data-str.json b/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-ai-data-str.json index 41edfdb936d4b..59f6a90bed2f1 100644 --- a/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-ai-data-str.json +++ b/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-ai-data-str.json @@ -1,5 +1,5 @@ { - "code" : "4260392550101", + "code" : "04260392550101", "errors" : [], "product" : { "_id" : "4260392550101", @@ -1169,7 +1169,7 @@ { "field" : { "id" : "code", - "value" : "4260392550101" + "value" : "04260392550101" }, "impact" : { "id" : "none", diff --git a/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-caret.json b/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-caret.json index f027166b62e17..79efa8eda8a66 100644 --- a/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-caret.json +++ b/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-caret.json @@ -1,5 +1,5 @@ { - "code" : "4260392550101", + "code" : "04260392550101", "errors" : [], "product" : { "_id" : "4260392550101", @@ -1169,7 +1169,7 @@ { "field" : { "id" : "code", - "value" : "4260392550101" + "value" : "04260392550101" }, "impact" : { "id" : "none", diff --git a/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-data-uri.json b/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-data-uri.json index f027166b62e17..79efa8eda8a66 100644 --- a/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-data-uri.json +++ b/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-data-uri.json @@ -1,5 +1,5 @@ { - "code" : "4260392550101", + "code" : "04260392550101", "errors" : [], "product" : { "_id" : "4260392550101", @@ -1169,7 +1169,7 @@ { "field" : { "id" : "code", - "value" : "4260392550101" + "value" : "04260392550101" }, "impact" : { "id" : "none", diff --git a/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-fnc1.json b/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-fnc1.json index 41edfdb936d4b..59f6a90bed2f1 100644 --- a/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-fnc1.json +++ b/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-fnc1.json @@ -1,5 +1,5 @@ { - "code" : "4260392550101", + "code" : "04260392550101", "errors" : [], "product" : { "_id" : "4260392550101", @@ -1169,7 +1169,7 @@ { "field" : { "id" : "code", - "value" : "4260392550101" + "value" : "04260392550101" }, "impact" : { "id" : "none", diff --git a/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-gs.json b/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-gs.json index f027166b62e17..79efa8eda8a66 100644 --- a/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-gs.json +++ b/tests/integration/expected_test_results/api_v3_product_read/get-existing-product-gs1-gs.json @@ -1,5 +1,5 @@ { - "code" : "4260392550101", + "code" : "04260392550101", "errors" : [], "product" : { "_id" : "4260392550101", @@ -1169,7 +1169,7 @@ { "field" : { "id" : "code", - "value" : "4260392550101" + "value" : "04260392550101" }, "impact" : { "id" : "none", diff --git a/tests/integration/expected_test_results/cors/get-api-v2.json b/tests/integration/expected_test_results/cors/get-api-v2.json index 339d526051d27..54e3931e5d20a 100644 --- a/tests/integration/expected_test_results/cors/get-api-v2.json +++ b/tests/integration/expected_test_results/cors/get-api-v2.json @@ -1,5 +1,5 @@ { - "code" : "0000002", + "code" : "00000002", "status" : 0, "status_verbose" : "product not found" } diff --git a/tests/integration/expected_test_results/data_quality_knowledge_panel/data-quality.json b/tests/integration/expected_test_results/data_quality_knowledge_panel/data-quality.json index 5bc29410efce8..22414cbb8217f 100644 --- a/tests/integration/expected_test_results/data_quality_knowledge_panel/data-quality.json +++ b/tests/integration/expected_test_results/data_quality_knowledge_panel/data-quality.json @@ -1,5 +1,5 @@ { - "code" : "200000000002", + "code" : "0200000000002", "product" : { "knowledge_panels" : { "contribution_card" : { diff --git a/tests/integration/expected_test_results/data_quality_knowledge_panel/no-data-quality.json b/tests/integration/expected_test_results/data_quality_knowledge_panel/no-data-quality.json index 963fab70f7001..e532ef647655d 100644 --- a/tests/integration/expected_test_results/data_quality_knowledge_panel/no-data-quality.json +++ b/tests/integration/expected_test_results/data_quality_knowledge_panel/no-data-quality.json @@ -1,5 +1,5 @@ { - "code" : "200000000001", + "code" : "0200000000001", "product" : { "knowledge_panels" : { "contribution_card" : { diff --git a/tests/integration/expected_test_results/export/export.csv b/tests/integration/expected_test_results/export/export.csv index 6cfc51b62d929..de39ad8994e70 100644 --- a/tests/integration/expected_test_results/export/export.csv +++ b/tests/integration/expected_test_results/export/export.csv @@ -19,7 +19,7 @@ Canola Oil, Water, Garbanzo Beans, Lupin (bean), Mustard Seeds, Grape Vinegar, L 3760178254021 fr Chocolat au lait France 100g 100g 148 kcal 9.2 g 5.4 g 13 g 13 g 3.4 g 148 kcal App - yuka, Apps 0 29161690 fr 100 % Almond Buter 227 g France 100g 100g 635 kcal 54.9 g 5.4 g 27.6 g 3.4 g 21 g 635 kcal App - yuka, Apps 0 3770013801303 fr Limonade Boissons et préparations de boissons, Boissons, Boissons gazeuses, Sodas, Limonades Bio France 100g 100g 31.2 kcal 1 g 1 g 7.5 g 7.5 g 1 g 0.7 g 31.2 kcal App - yuka, Apps 0 -25000044984 en Simply lemonade Simply Lemonade 8 OZA (240 ml) The Minute Maid Company Beverages and beverages preparations, Beverages, Carbonated drinks, Sodas, Lemonade, Sweetened beverages France, United States Contains pure filtered water, lemon juice, cane sugar, natural flavors. Contains pure filtered water, lemon juice, cane sugar, natural flavors. 100g 100g 50 kcal 0 g 0 g 12.5 g 11.67 g 0 g 0.0159 g 0.00636 g 50 kcal 2019-07-29T00:00:00Z Soda LI 678078 2019-07-29T00:00:00Z 2019-12-06T00:00:00Z App - yuka, Apps, Databases, database-usda, App - InFood 0 +0025000044984 en Simply lemonade Simply Lemonade 8 OZA (240 ml) The Minute Maid Company Beverages and beverages preparations, Beverages, Carbonated drinks, Sodas, Lemonade, Sweetened beverages France, United States Contains pure filtered water, lemon juice, cane sugar, natural flavors. Contains pure filtered water, lemon juice, cane sugar, natural flavors. 100g 100g 50 kcal 0 g 0 g 12.5 g 11.67 g 0 g 0.0159 g 0.00636 g 50 kcal 2019-07-29T00:00:00Z Soda LI 678078 2019-07-29T00:00:00Z 2019-12-06T00:00:00Z App - yuka, Apps, Databases, database-usda, App - InFood 0 80650904 fr Olio aromatizzato funghi e tartufo France 100g 100g App - yuka, Apps 0 9002355004345 de Tiroler Früchteküche Marillen 420 g Glas Tiroler Früchteküche "Pflanzliche Lebensmittel und Getränke, Pflanzliche Lebensmittel, Frucht- und gemüsebasierte Lebensmittel, Frühstücke, Brotaufstriche, Fruchtbasierte Lebensmittel, Pflanzliche Brotaufstriche, Süße Brotaufstriche, Konfitüren und Marmeladen, Konfitüren" Frankreich, Deutschland 100g 100g 224 kcal 0 g 0 g 54 g 53 g 0 g 0 g 0 g 224 kcal App - yuka, Apps 0 26281742 en Stawberry jam Strawberry conserve 500 g serving fr:bocal verre Grandessa Plant-based foods and beverages, Plant-based foods, Breakfasts, Spreads, Plant-based spreads, Sweet spreads, Fruit and vegetable preserves Made in Belgium France made in belgium Eggs, Milk serving 100g 171 kcal 0.1 g 0.1 g 9.9 g 9.9 g 0.1 g 2.54 g 1.016 g 171 kcal App - yuka, Apps 0 @@ -28,5 +28,6 @@ Canola Oil, Water, Garbanzo Beans, Lupin (bean), Mustard Seeds, Grape Vinegar, L 5601009974337 pt Iogurte morango banana Iogurte Líquido Meio Gordo com sumo e polpa de morango (4,4%) e polpa de banana (1%). 170 g Plástico, PE 7 - polietileno, HDPE 2 - Polietileno de alta densidade Pingo Doce, Gelgurte Laticínios, Alimentos Fermentados, Produtos lácteos fermentados, Sobremesas, Sobremesas de laticínios, en:Fermented dairy desserts, Iogurtes França, Portugal Pingo Doce Portugal Guarda, Portugal PT ILT 39 EC _Leite_ pasteurizado desnatado, _leite_ pasteurizado, açucar, sumo de morango reconstituído e polpa de morango, polpa de banana, _leite_ em pó magro, dextrose, aroma banana, aroma morango, corantes da fruta (antocianinas, beta-caroteno), amido de milho, espessantes (pectina amidada, goma xantana), fermentos lácticos, regulador de acidez (citrato trissódico), conservante da fruta (sorbato de potássio). Leite 100g 100g 78 kcal 1.4 g 0.9 g 13.7 g 11.1 g 0 g 2.7 g 0.1 g 0.04 g 78 kcal App - yuka, Apps 0 77000001 fr "Pâte à tartiner chocolat et noisettes" 300 g Alex Olivier "Petit-déjeuners, Produits à tartiner, Produits à tartiner sucrés, Pâtes à tartiner, Pâtes à tartiner aux noisettes, Pâtes à tartiner au chocolat, Pâtes à tartiner aux noisettes et au cacao" France 100g 100g 49 % 0 8712423020221 fr The Tasty Decaf Unilever France 100g 100g 1 kcal 0 g 0 g 0 g 0 g 0 g 0 g 0 g 1 kcal App - yuka, Apps 0 -71464240608 en 100% fruit juice smoothie, green goodness Green goodness smoothie 8 OZA (240 ml) Bolthouse Farms Wm. Bolthouse Farms Inc. Beverages and beverages preparations, Plant-based foods and beverages, Beverages, Plant-based beverages France, United States Pineapple juice from concentrate (water, pineapple juice concentrate), apple juice from concentrate (water, apple juice concentrate), mango puree from concentrate (water, mango puree concentrate), banana puree, kiwi puree, spirulina powder, natural flavor, green tea extract, spinach powder, broccoli powder, barley grass powder, wheat grass powder, garlic powder, jerusalem artichoke powder, nova scotia dulse powder. 100g 100g 58 kcal 0 g 0 g 13.75 g 12.5 g 0.8 g 0.83 g 30 mg 12 mg 58 kcal 0 g 0 mg 417 IU 12.5 mg 0.071 mg 0.125 mg 0.62 µg 175 mg 17 mg 0.45 mg 1 mg 2018-02-14T00:00:00Z Fruit & Vegetable Juice, Nectars & Fruit Drinks LI 653042 2018-02-14T00:00:00Z 2019-12-06T00:00:00Z App - yuka, Apps, Databases, database-usda 0 +0071464240608 en 100% fruit juice smoothie, green goodness Green goodness smoothie 8 OZA (240 ml) Bolthouse Farms Wm. Bolthouse Farms Inc. Beverages and beverages preparations, Plant-based foods and beverages, Beverages, Plant-based beverages France, United States Pineapple juice from concentrate (water, pineapple juice concentrate), apple juice from concentrate (water, apple juice concentrate), mango puree from concentrate (water, mango puree concentrate), banana puree, kiwi puree, spirulina powder, natural flavor, green tea extract, spinach powder, broccoli powder, barley grass powder, wheat grass powder, garlic powder, jerusalem artichoke powder, nova scotia dulse powder. 100g 100g 58 kcal 0 g 0 g 13.75 g 12.5 g 0.8 g 0.83 g 30 mg 12 mg 58 kcal 0 g 0 mg 417 IU 12.5 mg 0.071 mg 0.125 mg 0.62 µg 175 mg 17 mg 0.45 mg 1 mg 2018-02-14T00:00:00Z Fruit & Vegetable Juice, Nectars & Fruit Drinks LI 653042 2018-02-14T00:00:00Z 2019-12-06T00:00:00Z App - yuka, Apps, Databases, database-usda 0 +04083637 fr Lait concentré non sucré entier 410 g Régilait Produits laitiers, Laits concentrés, Laits concentrés non sucrés, Laits entiers concentrés Fabriqué en Allemagne France DE RP-25 EC Lait entier (origine UE), stabilisants : phosphates de sodium, carraghénanes Lait 100g 100g 132 kcal 7.5 g 5 g 10 g 10 g 6.1 g 0.25 g 0.1 g 132 kcal App - yuka, Apps 0 0850032917148 en Tulú Drinks - Strawberry Flavor Aloe Drink with Strawberry Flavor 500 ml 240 ml Plastic Tulú, Tulú Drinks Colombia's Best Beverages and beverages preparations, Plant-based foods and beverages, Beverages, Plant-based beverages, Fruit-based beverages, Non-alcoholic beverages, Aloe Vera drinks Vegetarian, No preservatives, Vegan Puerto Rico Colombia Water, Aloe Vera Pulp, Sugar, Citric Acid, Sodium Citrate, Aloe Artifical Flavor, Calcium Lactate, Strawberry Artificial Flavor, Ascorbic Acid, Gellan Gum, EDTA, Sucralose, FD&C Red No. 40 None None serving 100g 50 kcal 0 g 0 g 12 g 12 g 0 g 0 g 100 mg 40 mg 50 kcal 0 diff --git a/tests/integration/expected_test_results/export/rows/25000044984.json b/tests/integration/expected_test_results/export/rows/0025000044984.json similarity index 99% rename from tests/integration/expected_test_results/export/rows/25000044984.json rename to tests/integration/expected_test_results/export/rows/0025000044984.json index 11b1df896a934..31e40e3850d62 100644 --- a/tests/integration/expected_test_results/export/rows/25000044984.json +++ b/tests/integration/expected_test_results/export/rows/0025000044984.json @@ -11,7 +11,7 @@ "categories" : "Beverages and beverages preparations, Beverages, Carbonated drinks, Sodas, Lemonade, Sweetened beverages", "cholesterol_unit" : "", "cholesterol_value" : "", - "code" : "25000044984", + "code" : "0025000044984", "conservation_conditions_fr" : "", "countries" : "France, United States", "customer_service_fr" : "", diff --git a/tests/integration/expected_test_results/export/rows/71464240608.json b/tests/integration/expected_test_results/export/rows/0071464240608.json similarity index 99% rename from tests/integration/expected_test_results/export/rows/71464240608.json rename to tests/integration/expected_test_results/export/rows/0071464240608.json index c60c73500153a..2ab76c9ba1a35 100644 --- a/tests/integration/expected_test_results/export/rows/71464240608.json +++ b/tests/integration/expected_test_results/export/rows/0071464240608.json @@ -11,7 +11,7 @@ "categories" : "Beverages and beverages preparations, Plant-based foods and beverages, Beverages, Plant-based beverages", "cholesterol_unit" : "mg", "cholesterol_value" : "0", - "code" : "71464240608", + "code" : "0071464240608", "conservation_conditions_fr" : "", "countries" : "France, United States", "customer_service_fr" : "", diff --git a/tests/integration/expected_test_results/export/rows/04083637.json b/tests/integration/expected_test_results/export/rows/04083637.json new file mode 100644 index 0000000000000..a88a2c2dfa8b1 --- /dev/null +++ b/tests/integration/expected_test_results/export/rows/04083637.json @@ -0,0 +1,123 @@ +{ + "abbreviated_product_name_en" : "", + "abbreviated_product_name_fr" : "", + "allergens" : "Lait", + "brand_owner" : "", + "brands" : "Régilait", + "calcium_unit" : "", + "calcium_value" : "", + "carbohydrates_unit" : "g", + "carbohydrates_value" : "10", + "categories" : "Produits laitiers, Laits concentrés, Laits concentrés non sucrés, Laits entiers concentrés", + "cholesterol_unit" : "", + "cholesterol_value" : "", + "code" : "04083637", + "conservation_conditions_fr" : "", + "countries" : "France", + "customer_service_fr" : "", + "data_sources" : "App - yuka, Apps", + "emb_codes" : "DE RP-25 EC", + "energy-kcal_unit" : "kcal", + "energy-kcal_value" : "132", + "energy-kj_unit" : "", + "energy-kj_value" : "", + "energy_unit" : "kcal", + "energy_value" : "132", + "fat_unit" : "g", + "fat_value" : "7.5", + "fiber_unit" : "", + "fiber_value" : "", + "fruits-vegetables-nuts-estimate_unit" : "", + "fruits-vegetables-nuts-estimate_value" : "", + "generic_name_en" : "", + "generic_name_fr" : "", + "generic_name_pt" : "", + "ingredients_text_en" : "", + "ingredients_text_es" : "", + "ingredients_text_fr" : "Lait entier (origine UE), stabilisants : phosphates de sodium, carraghénanes", + "ingredients_text_pt" : "", + "iron_unit" : "", + "iron_value" : "", + "labels" : "Fabriqué en Allemagne", + "lc" : "fr", + "link" : "", + "manganese_unit" : "", + "manganese_value" : "", + "manufacturing_places" : "", + "monounsaturated-fat_unit" : "", + "monounsaturated-fat_value" : "", + "nutrition_data_per" : "100g", + "nutrition_data_prepared_per" : "100g", + "obsolete" : "0", + "origin_fr" : "", + "origins" : "", + "packaging" : "", + "packaging_1_material" : "", + "packaging_1_number_of_units" : "", + "packaging_1_quantity_per_unit" : "", + "packaging_1_shape" : "", + "packaging_1_weight_measured" : "", + "packaging_1_weight_specified" : "", + "packaging_2_material" : "", + "packaging_2_number_of_units" : "", + "packaging_2_shape" : "", + "packaging_2_weight_measured" : "", + "polyunsaturated-fat_unit" : "", + "polyunsaturated-fat_value" : "", + "potassium_unit" : "", + "potassium_value" : "", + "preparation_fr" : "", + "producer_product_id" : "", + "producer_version_id" : "", + "product_name_de" : "", + "product_name_en" : "", + "product_name_es" : "", + "product_name_fr" : "Lait concentré non sucré entier", + "product_name_pt" : "", + "proteins_unit" : "g", + "proteins_value" : "6.1", + "quantity" : "410 g", + "salt_unit" : "g", + "salt_value" : "0.25", + "saturated-fat_unit" : "g", + "saturated-fat_value" : "5", + "serving_size" : "", + "sodium_unit" : "g", + "sodium_value" : "0.1", + "sources_fields:org-database-usda:available_date" : "", + "sources_fields:org-database-usda:fdc_category" : "", + "sources_fields:org-database-usda:fdc_data_source" : "", + "sources_fields:org-database-usda:fdc_id" : "", + "sources_fields:org-database-usda:modified_date" : "", + "sources_fields:org-database-usda:publication_date" : "", + "sources_fields:org-gs1:gln" : "", + "sources_fields:org-gs1:gpcCategoryCode" : "", + "sources_fields:org-gs1:gpcCategoryName" : "", + "sources_fields:org-gs1:isAllergenRelevantDataProvided" : "", + "sources_fields:org-gs1:lastChangeDateTime" : "", + "sources_fields:org-gs1:partyName" : "", + "sources_fields:org-gs1:productionVariantDescription" : "", + "sources_fields:org-gs1:publicationDateTime" : "", + "stores" : "", + "sugars_unit" : "g", + "sugars_value" : "10", + "traces" : "", + "trans-fat_unit" : "", + "trans-fat_value" : "", + "vitamin-a_unit" : "", + "vitamin-a_value" : "", + "vitamin-b12_unit" : "", + "vitamin-b12_value" : "", + "vitamin-b1_unit" : "", + "vitamin-b1_value" : "", + "vitamin-b2_unit" : "", + "vitamin-b2_value" : "", + "vitamin-b6_unit" : "", + "vitamin-b6_value" : "", + "vitamin-b9_unit" : "", + "vitamin-b9_value" : "", + "vitamin-c_unit" : "", + "vitamin-c_value" : "", + "vitamin-pp_unit" : "", + "vitamin-pp_value" : "" +} diff --git a/tests/integration/expected_test_results/export_more_fields/export_more_fields.csv b/tests/integration/expected_test_results/export_more_fields/export_more_fields.csv index 07968c8afbb67..797f5a4e454fd 100644 --- a/tests/integration/expected_test_results/export_more_fields/export_more_fields.csv +++ b/tests/integration/expected_test_results/export_more_fields/export_more_fields.csv @@ -19,7 +19,7 @@ Canola Oil, Water, Garbanzo Beans, Lupin (bean), Mustard Seeds, Grape Vinegar, L 3760178254021 fr Chocolat au lait France en:france 100g 100g 148 kcal 9.2 g 5.4 g 13 g 13 g 3.4 g 148 kcal unknown unknown unknown 1 -5 -15 1 0 App - yuka, Apps 0 29161690 fr 100 % Almond Buter 227 g France en:france 100g 100g 635 kcal 54.9 g 5.4 g 27.6 g 3.4 g 21 g 635 kcal unknown unknown unknown 1 -5 -15 1 0 App - yuka, Apps 0 3770013801303 fr Limonade Boissons et préparations de boissons, Boissons, Boissons gazeuses, Sodas, Limonades en:beverages-and-beverages-preparations,en:beverages,en:carbonated-drinks,en:sodas,en:lemonade Bio en:organic France en:france 100g 100g 31.2 kcal 1 g 1 g 7.5 g 7.5 g 1 g 0.7 g 31.2 kcal en:sweetened-beverages en:beverages,en:sweetened-beverages unknown unknown not-applicable 1 -5 -15 1 0 App - yuka, Apps 0 -25000044984 en Simply lemonade Simply Lemonade 8 OZA (240 ml) The Minute Maid Company Beverages and beverages preparations, Beverages, Carbonated drinks, Sodas, Lemonade, Sweetened beverages en:beverages-and-beverages-preparations,en:beverages,en:carbonated-drinks,en:sodas,en:lemonade,en:sweetened-beverages France, United States en:france,en:united-states Contains pure filtered water, lemon juice, cane sugar, natural flavors. Contains pure filtered water, lemon juice, cane sugar, natural flavors. 100g 100g 50 kcal 0 g 0 g 12.5 g 11.67 g 0 g 0.0159 g 0.00636 g 50 kcal en:sweetened-beverages en:beverages,en:sweetened-beverages 4 en:4-ultra-processed-food-and-drink-products e 15 not-applicable 1 -5 -15 1 0 2019-07-29T00:00:00Z Soda LI 678078 2019-07-29T00:00:00Z 2019-12-06T00:00:00Z App - yuka, Apps, Databases, database-usda, App - InFood 0 +0025000044984 en Simply lemonade Simply Lemonade 8 OZA (240 ml) The Minute Maid Company Beverages and beverages preparations, Beverages, Carbonated drinks, Sodas, Lemonade, Sweetened beverages en:beverages-and-beverages-preparations,en:beverages,en:carbonated-drinks,en:sodas,en:lemonade,en:sweetened-beverages France, United States en:france,en:united-states Contains pure filtered water, lemon juice, cane sugar, natural flavors. Contains pure filtered water, lemon juice, cane sugar, natural flavors. 100g 100g 50 kcal 0 g 0 g 12.5 g 11.67 g 0 g 0.0159 g 0.00636 g 50 kcal en:sweetened-beverages en:beverages,en:sweetened-beverages 4 en:4-ultra-processed-food-and-drink-products e 15 not-applicable 1 -5 -15 1 0 2019-07-29T00:00:00Z Soda LI 678078 2019-07-29T00:00:00Z 2019-12-06T00:00:00Z App - yuka, Apps, Databases, database-usda, App - InFood 0 80650904 fr Olio aromatizzato funghi e tartufo France en:france 100g 100g unknown unknown unknown 1 -5 -15 1 0 App - yuka, Apps 0 9002355004345 de Tiroler Früchteküche Marillen 420 g Glas en:glass Tiroler Früchteküche tiroler-fruchtekuche "Pflanzliche Lebensmittel und Getränke, Pflanzliche Lebensmittel, Frucht- und gemüsebasierte Lebensmittel, Frühstücke, Brotaufstriche, Fruchtbasierte Lebensmittel, Pflanzliche Brotaufstriche, Süße Brotaufstriche, Konfitüren und Marmeladen, Konfitüren" en:plant-based-foods-and-beverages,en:plant-based-foods,en:fruits-and-vegetables-based-foods,en:breakfasts,en:spreads,en:fruits-based-foods,en:plant-based-spreads,en:sweet-spreads,en:fruit-and-vegetable-preserves,en:jams Frankreich, Deutschland en:france,en:germany 100g 100g 224 kcal 0 g 0 g 54 g 53 g 0 g 0 g 0 g 224 kcal en:sweets en:sugary-snacks,en:sweets unknown d 11 b 60 1 31024 -5 -15 1 0 App - yuka, Apps 0 26281742 en Stawberry jam Strawberry conserve 500 g serving fr:bocal verre fr:bocal-verre Grandessa grandessa Plant-based foods and beverages, Plant-based foods, Breakfasts, Spreads, Plant-based spreads, Sweet spreads, Fruit and vegetable preserves en:plant-based-foods-and-beverages,en:plant-based-foods,en:breakfasts,en:spreads,en:plant-based-spreads,en:sweet-spreads,en:fruit-and-vegetable-preserves Made in Belgium en:made-in-belgium France en:france made in belgium made-in-belgium Eggs, Milk en:eggs,en:milk serving 100g 171 kcal 0.1 g 0.1 g 9.9 g 9.9 g 0.1 g 2.54 g 1.016 g 171 kcal en:sweets en:sugary-snacks,en:sweets unknown unknown unknown 1 -5 -15 1 0 App - yuka, Apps 0 @@ -28,5 +28,6 @@ Canola Oil, Water, Garbanzo Beans, Lupin (bean), Mustard Seeds, Grape Vinegar, L 5601009974337 pt Iogurte morango banana Iogurte Líquido Meio Gordo com sumo e polpa de morango (4,4%) e polpa de banana (1%). 170 g Plástico, PE 7 - polietileno, HDPE 2 - Polietileno de alta densidade en:plastic,en:pe-7-polyethylene,en:hdpe-2-high-density-polyethylene Pingo Doce, Gelgurte pingo-doce,gelgurte Laticínios, Alimentos Fermentados, Produtos lácteos fermentados, Sobremesas, Sobremesas de laticínios, en:Fermented dairy desserts, Iogurtes en:dairies,en:fermented-foods,en:fermented-milk-products,en:desserts,en:dairy-desserts,en:fermented-dairy-desserts,en:yogurts França, Portugal en:france,en:portugal Pingo Doce pingo-doce Portugal en:portugal Guarda, Portugal guarda,portugal PT ILT 39 EC pt-ilt-39-ec _Leite_ pasteurizado desnatado, _leite_ pasteurizado, açucar, sumo de morango reconstituído e polpa de morango, polpa de banana, _leite_ em pó magro, dextrose, aroma banana, aroma morango, corantes da fruta (antocianinas, beta-caroteno), amido de milho, espessantes (pectina amidada, goma xantana), fermentos lácticos, regulador de acidez (citrato trissódico), conservante da fruta (sorbato de potássio). Leite en:milk 100g 100g 78 kcal 1.4 g 0.9 g 13.7 g 11.1 g 0 g 2.7 g 0.1 g 0.04 g 78 kcal en:milk-and-yogurt en:milk-and-dairy-products,en:milk-and-yogurt 4 en:4-ultra-processed-food-and-drink-products b 1 b 79 1 19593 8 -15 1 0 App - yuka, Apps 0 77000001 fr "Pâte à tartiner chocolat et noisettes" 300 g Alex Olivier alex-olivier "Petit-déjeuners, Produits à tartiner, Produits à tartiner sucrés, Pâtes à tartiner, Pâtes à tartiner aux noisettes, Pâtes à tartiner au chocolat, Pâtes à tartiner aux noisettes et au cacao" en:breakfasts,en:spreads,en:sweet-spreads,fr:pates-a-tartiner,en:hazelnut-spreads,en:chocolate-spreads,en:cocoa-and-hazelnuts-spreads France en:france 100g 100g 49 % en:sweets en:sugary-snacks,en:sweets unknown unknown d 31 1 31032 -5 -15 1 0 0 8712423020221 fr The Tasty Decaf Unilever unilever France en:france 100g 100g 1 kcal 0 g 0 g 0 g 0 g 0 g 0 g 0 g 1 kcal unknown unknown unknown 1 -5 -15 1 0 App - yuka, Apps 0 -71464240608 en 100% fruit juice smoothie, green goodness Green goodness smoothie 8 OZA (240 ml) Bolthouse Farms bolthouse-farms Wm. Bolthouse Farms Inc. Beverages and beverages preparations, Plant-based foods and beverages, Beverages, Plant-based beverages en:beverages-and-beverages-preparations,en:plant-based-foods-and-beverages,en:beverages,en:plant-based-beverages France, United States en:france,en:united-states Pineapple juice from concentrate (water, pineapple juice concentrate), apple juice from concentrate (water, apple juice concentrate), mango puree from concentrate (water, mango puree concentrate), banana puree, kiwi puree, spirulina powder, natural flavor, green tea extract, spinach powder, broccoli powder, barley grass powder, wheat grass powder, garlic powder, jerusalem artichoke powder, nova scotia dulse powder. 100g 100g 58 kcal 0 g 0 g 13.75 g 12.5 g 0.8 g 0.83 g 30 mg 12 mg 58 kcal 0 g 0 mg 417 IU 12.5 mg 0.071 mg 0.125 mg 0.62 µg 175 mg 17 mg 0.45 mg 1 mg en:unsweetened-beverages en:beverages,en:unsweetened-beverages 4 en:4-ultra-processed-food-and-drink-products d 8 unknown 1 -5 -15 1 0 2018-02-14T00:00:00Z Fruit & Vegetable Juice, Nectars & Fruit Drinks LI 653042 2018-02-14T00:00:00Z 2019-12-06T00:00:00Z App - yuka, Apps, Databases, database-usda 0 +0071464240608 en 100% fruit juice smoothie, green goodness Green goodness smoothie 8 OZA (240 ml) Bolthouse Farms bolthouse-farms Wm. Bolthouse Farms Inc. Beverages and beverages preparations, Plant-based foods and beverages, Beverages, Plant-based beverages en:beverages-and-beverages-preparations,en:plant-based-foods-and-beverages,en:beverages,en:plant-based-beverages France, United States en:france,en:united-states Pineapple juice from concentrate (water, pineapple juice concentrate), apple juice from concentrate (water, apple juice concentrate), mango puree from concentrate (water, mango puree concentrate), banana puree, kiwi puree, spirulina powder, natural flavor, green tea extract, spinach powder, broccoli powder, barley grass powder, wheat grass powder, garlic powder, jerusalem artichoke powder, nova scotia dulse powder. 100g 100g 58 kcal 0 g 0 g 13.75 g 12.5 g 0.8 g 0.83 g 30 mg 12 mg 58 kcal 0 g 0 mg 417 IU 12.5 mg 0.071 mg 0.125 mg 0.62 µg 175 mg 17 mg 0.45 mg 1 mg en:unsweetened-beverages en:beverages,en:unsweetened-beverages 4 en:4-ultra-processed-food-and-drink-products d 8 unknown 1 -5 -15 1 0 2018-02-14T00:00:00Z Fruit & Vegetable Juice, Nectars & Fruit Drinks LI 653042 2018-02-14T00:00:00Z 2019-12-06T00:00:00Z App - yuka, Apps, Databases, database-usda 0 +04083637 fr Lait concentré non sucré entier 410 g Régilait regilait Produits laitiers, Laits concentrés, Laits concentrés non sucrés, Laits entiers concentrés en:dairies,en:evaporated-milks,en:evaporated-milks-without-sugar,fr:laits-entiers-concentres Fabriqué en Allemagne en:made-in-germany France en:france DE RP-25 EC de-rp-25-ec Lait entier (origine UE), stabilisants : phosphates de sodium, carraghénanes Lait en:milk 100g 100g 132 kcal 7.5 g 5 g 10 g 10 g 6.1 g 0.25 g 0.1 g 132 kcal 4 en:4-ultra-processed-food-and-drink-products c 5 unknown 1 -4 -15 1 0 App - yuka, Apps 0 0850032917148 en Tulú Drinks - Strawberry Flavor Aloe Drink with Strawberry Flavor 500 ml 240 ml Plastic en:plastic Tulú, Tulú Drinks tulu,tulu-drinks Colombia's Best Beverages and beverages preparations, Plant-based foods and beverages, Beverages, Plant-based beverages, Fruit-based beverages, Non-alcoholic beverages, Aloe Vera drinks en:beverages-and-beverages-preparations,en:plant-based-foods-and-beverages,en:beverages,en:plant-based-beverages,en:fruit-based-beverages,en:non-alcoholic-beverages,en:aloe-vera-drinks Vegetarian, No preservatives, Vegan en:vegetarian,en:no-preservatives,en:vegan Puerto Rico en:puerto-rico Colombia en:colombia Water, Aloe Vera Pulp, Sugar, Citric Acid, Sodium Citrate, Aloe Artifical Flavor, Calcium Lactate, Strawberry Artificial Flavor, Ascorbic Acid, Gellan Gum, EDTA, Sucralose, FD&C Red No. 40 None en:none None en:none serving 100g 50 kcal 0 g 0 g 12 g 12 g 0 g 0 g 100 mg 40 mg 50 kcal en:sweetened-beverages en:beverages,en:sweetened-beverages 4 en:4-ultra-processed-food-and-drink-products d 7 c 48 1 18309 -1 -15 1 0 0 diff --git a/tests/integration/expected_test_results/export_more_fields/rows/25000044984.json b/tests/integration/expected_test_results/export_more_fields/rows/0025000044984.json similarity index 99% rename from tests/integration/expected_test_results/export_more_fields/rows/25000044984.json rename to tests/integration/expected_test_results/export_more_fields/rows/0025000044984.json index 8bbd4560221b8..127c0c95907a2 100644 --- a/tests/integration/expected_test_results/export_more_fields/rows/25000044984.json +++ b/tests/integration/expected_test_results/export_more_fields/rows/0025000044984.json @@ -14,7 +14,7 @@ "categories_tags" : "en:beverages-and-beverages-preparations,en:beverages,en:carbonated-drinks,en:sodas,en:lemonade,en:sweetened-beverages", "cholesterol_unit" : "", "cholesterol_value" : "", - "code" : "25000044984", + "code" : "0025000044984", "conservation_conditions_fr" : "", "countries" : "France, United States", "countries_tags" : "en:france,en:united-states", diff --git a/tests/integration/expected_test_results/export_more_fields/rows/71464240608.json b/tests/integration/expected_test_results/export_more_fields/rows/0071464240608.json similarity index 99% rename from tests/integration/expected_test_results/export_more_fields/rows/71464240608.json rename to tests/integration/expected_test_results/export_more_fields/rows/0071464240608.json index 9fa2ac973dff6..ff7a2c25b1b17 100644 --- a/tests/integration/expected_test_results/export_more_fields/rows/71464240608.json +++ b/tests/integration/expected_test_results/export_more_fields/rows/0071464240608.json @@ -14,7 +14,7 @@ "categories_tags" : "en:beverages-and-beverages-preparations,en:plant-based-foods-and-beverages,en:beverages,en:plant-based-beverages", "cholesterol_unit" : "mg", "cholesterol_value" : "0", - "code" : "71464240608", + "code" : "0071464240608", "conservation_conditions_fr" : "", "countries" : "France, United States", "countries_tags" : "en:france,en:united-states", diff --git a/tests/integration/expected_test_results/export_more_fields/rows/04083637.json b/tests/integration/expected_test_results/export_more_fields/rows/04083637.json new file mode 100644 index 0000000000000..9f2ffe8ecdd95 --- /dev/null +++ b/tests/integration/expected_test_results/export_more_fields/rows/04083637.json @@ -0,0 +1,149 @@ +{ + "abbreviated_product_name_en" : "", + "abbreviated_product_name_fr" : "", + "allergens" : "Lait", + "allergens_tags" : "en:milk", + "brand_owner" : "", + "brands" : "Régilait", + "brands_tags" : "regilait", + "calcium_unit" : "", + "calcium_value" : "", + "carbohydrates_unit" : "g", + "carbohydrates_value" : "10", + "categories" : "Produits laitiers, Laits concentrés, Laits concentrés non sucrés, Laits entiers concentrés", + "categories_tags" : "en:dairies,en:evaporated-milks,en:evaporated-milks-without-sugar,fr:laits-entiers-concentres", + "cholesterol_unit" : "", + "cholesterol_value" : "", + "code" : "04083637", + "conservation_conditions_fr" : "", + "countries" : "France", + "countries_tags" : "en:france", + "customer_service_fr" : "", + "data_sources" : "App - yuka, Apps", + "emb_codes" : "DE RP-25 EC", + "emb_codes_tags" : "de-rp-25-ec", + "energy-kcal_unit" : "kcal", + "energy-kcal_value" : "132", + "energy-kj_unit" : "", + "energy-kj_value" : "", + "energy_unit" : "kcal", + "energy_value" : "132", + "fat_unit" : "g", + "fat_value" : "7.5", + "fiber_unit" : "", + "fiber_value" : "", + "fruits-vegetables-nuts-estimate_unit" : "", + "fruits-vegetables-nuts-estimate_value" : "", + "generic_name_en" : "", + "generic_name_fr" : "", + "generic_name_pt" : "", + "ingredients_text_en" : "", + "ingredients_text_es" : "", + "ingredients_text_fr" : "Lait entier (origine UE), stabilisants : phosphates de sodium, carraghénanes", + "ingredients_text_pt" : "", + "iron_unit" : "", + "iron_value" : "", + "labels" : "Fabriqué en Allemagne", + "labels_tags" : "en:made-in-germany", + "lc" : "fr", + "link" : "", + "manganese_unit" : "", + "manganese_value" : "", + "manufacturing_places" : "", + "manufacturing_places_tags" : "", + "monounsaturated-fat_unit" : "", + "monounsaturated-fat_value" : "", + "nutrition_data_per" : "100g", + "nutrition_data_prepared_per" : "100g", + "obsolete" : "0", + "off:ecoscore_data.adjustments.origins_of_ingredients.value" : "-4", + "off:ecoscore_data.adjustments.packaging.non_recyclable_and_non_biodegradable_materials" : "1", + "off:ecoscore_data.adjustments.packaging.value" : "-15", + "off:ecoscore_data.adjustments.production_system.value" : "0", + "off:ecoscore_data.adjustments.threatened_species.value" : "", + "off:ecoscore_data.agribalyse.code" : "", + "off:ecoscore_data.missing_key_data" : "1", + "off:ecoscore_grade" : "unknown", + "off:ecoscore_score" : "", + "off:food_groups" : "", + "off:food_groups_tags" : "", + "off:nova_groups" : "4", + "off:nova_groups_tags" : "en:4-ultra-processed-food-and-drink-products", + "off:nutriscore_grade" : "c", + "off:nutriscore_score" : "5", + "origin_fr" : "", + "origins" : "", + "origins_tags" : "", + "packaging" : "", + "packaging_1_material" : "", + "packaging_1_number_of_units" : "", + "packaging_1_quantity_per_unit" : "", + "packaging_1_shape" : "", + "packaging_1_weight_measured" : "", + "packaging_1_weight_specified" : "", + "packaging_2_material" : "", + "packaging_2_number_of_units" : "", + "packaging_2_shape" : "", + "packaging_2_weight_measured" : "", + "packaging_tags" : "", + "polyunsaturated-fat_unit" : "", + "polyunsaturated-fat_value" : "", + "potassium_unit" : "", + "potassium_value" : "", + "preparation_fr" : "", + "producer_product_id" : "", + "producer_version_id" : "", + "product_name_de" : "", + "product_name_en" : "", + "product_name_es" : "", + "product_name_fr" : "Lait concentré non sucré entier", + "product_name_pt" : "", + "proteins_unit" : "g", + "proteins_value" : "6.1", + "quantity" : "410 g", + "salt_unit" : "g", + "salt_value" : "0.25", + "saturated-fat_unit" : "g", + "saturated-fat_value" : "5", + "serving_size" : "", + "sodium_unit" : "g", + "sodium_value" : "0.1", + "sources_fields:org-database-usda:available_date" : "", + "sources_fields:org-database-usda:fdc_category" : "", + "sources_fields:org-database-usda:fdc_data_source" : "", + "sources_fields:org-database-usda:fdc_id" : "", + "sources_fields:org-database-usda:modified_date" : "", + "sources_fields:org-database-usda:publication_date" : "", + "sources_fields:org-gs1:gln" : "", + "sources_fields:org-gs1:gpcCategoryCode" : "", + "sources_fields:org-gs1:gpcCategoryName" : "", + "sources_fields:org-gs1:isAllergenRelevantDataProvided" : "", + "sources_fields:org-gs1:lastChangeDateTime" : "", + "sources_fields:org-gs1:partyName" : "", + "sources_fields:org-gs1:productionVariantDescription" : "", + "sources_fields:org-gs1:publicationDateTime" : "", + "stores" : "", + "stores_tags" : "", + "sugars_unit" : "g", + "sugars_value" : "10", + "traces" : "", + "traces_tags" : "", + "trans-fat_unit" : "", + "trans-fat_value" : "", + "vitamin-a_unit" : "", + "vitamin-a_value" : "", + "vitamin-b12_unit" : "", + "vitamin-b12_value" : "", + "vitamin-b1_unit" : "", + "vitamin-b1_value" : "", + "vitamin-b2_unit" : "", + "vitamin-b2_value" : "", + "vitamin-b6_unit" : "", + "vitamin-b6_value" : "", + "vitamin-b9_unit" : "", + "vitamin-b9_value" : "", + "vitamin-c_unit" : "", + "vitamin-c_value" : "", + "vitamin-pp_unit" : "", + "vitamin-pp_value" : "" +} diff --git a/tests/integration/expected_test_results/product_read/get-existing-product.html b/tests/integration/expected_test_results/product_read/get-existing-product.html index 0072bb00d2609..899b09ac5f4a2 100644 --- a/tests/integration/expected_test_results/product_read/get-existing-product.html +++ b/tests/integration/expected_test_results/product_read/get-existing-product.html @@ -10,7 +10,7 @@ - + @@ -25,7 +25,7 @@ - + @@ -70,7 +70,7 @@ - +
@@ -363,7 +363,7 @@ - + @@ -379,7 +379,7 @@

Some product - 100 g
- + edit @@ -389,7 +389,7 @@

Some product - 100 g