From 4833b85c189064c81531294f4bb37b8277ffc9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Thu, 4 Jan 2024 16:27:04 +0100 Subject: [PATCH 01/21] TDD: Add test cases for boolean field --- frictionless/fields/__spec__/test_boolean.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frictionless/fields/__spec__/test_boolean.py b/frictionless/fields/__spec__/test_boolean.py index 72d7954dda..b739a5f661 100644 --- a/frictionless/fields/__spec__/test_boolean.py +++ b/frictionless/fields/__spec__/test_boolean.py @@ -32,7 +32,9 @@ ("default", "3.14", None, {}), ("default", "", None, {}), ("default", "Yes", None, {"trueValues": ["yes"]}), + ("default", "true", True, {"trueValues": ["yes"]}), ("default", "No", None, {"falseValues": ["no"]}), + ("default", "false", False, {"falseValues": ["no"]}), ], ) def test_boolean_read_cell(format, source, target, options): From 581279bc5848453c4789a85999c5d259c30080f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Fri, 5 Jan 2024 10:23:27 +0100 Subject: [PATCH 02/21] TDD: tests passe: fix Field with boolean type creation from descriptor to use default true values and default false values --- frictionless/schema/field.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/frictionless/schema/field.py b/frictionless/schema/field.py index 75995174e3..e1b13bf1ff 100644 --- a/frictionless/schema/field.py +++ b/frictionless/schema/field.py @@ -3,9 +3,11 @@ import decimal import re from functools import partial -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, Optional, Pattern +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, Optional +from typing import Pattern, Union import attrs +from typing_extensions import Self from .. import errors, settings from ..exception import FrictionlessException @@ -194,6 +196,21 @@ def value_writer(cell: Any): return value_writer + @classmethod + def from_descriptor( # type: ignore + cls, + descriptor: Union[IDescriptor, str], + **options: Any, + ) -> Self: + if isinstance(descriptor, dict): + if "trueValues" in descriptor.keys(): + assert descriptor["type"] == "boolean" + descriptor["trueValues"] = descriptor["trueValues"] + settings.DEFAULT_TRUE_VALUES + if "falseValues" in descriptor.keys(): + assert descriptor["type"] == "boolean" + descriptor["falseValues"] = descriptor["falseValues"] + settings.DEFAULT_FALSE_VALUES + return super().from_descriptor(descriptor) + # Metadata metadata_type = "field" From 17f74c17c64ecf58ac26c108482c6e3504aa3d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Fri, 5 Jan 2024 15:18:54 +0100 Subject: [PATCH 03/21] TDD: Add test cases for boolean field customised with 'trueValues' and 'falseValues' in a schema descriptor --- frictionless/fields/__spec__/test_boolean.py | 26 +++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/frictionless/fields/__spec__/test_boolean.py b/frictionless/fields/__spec__/test_boolean.py index b739a5f661..9d4f7a4863 100644 --- a/frictionless/fields/__spec__/test_boolean.py +++ b/frictionless/fields/__spec__/test_boolean.py @@ -1,6 +1,6 @@ import pytest -from frictionless import Field +from frictionless import Field, Schema # General @@ -43,3 +43,27 @@ def test_boolean_read_cell(format, source, target, options): field = Field.from_descriptor(descriptor) cell, notes = field.read_cell(source) assert cell == target + + +def test_boolean_from_schema_descriptor_read_cell(): + schema = Schema.from_descriptor( + { + "fields": [ + {"name": "IsTrue", "type": "boolean", "trueValues": ["yes"], "falseValues": ["no"]} + ] + } + ) + + source = "true" + target = True + + fields = schema.fields + cell, notes = fields[0].read_cell(source) + assert cell == target + + source = "false" + target = False + + fields = schema.fields + cell, notes = fields[0].read_cell(source) + assert cell == target From 7aa1cde2620d78683dca6670acacb12fd953c2a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Fri, 5 Jan 2024 15:22:56 +0100 Subject: [PATCH 04/21] TDD: test pass, fix BooleanField creation from a schema descriptor to use default true values and default false values --- frictionless/fields/boolean.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/frictionless/fields/boolean.py b/frictionless/fields/boolean.py index 6d78984fe1..6e2ca211fe 100644 --- a/frictionless/fields/boolean.py +++ b/frictionless/fields/boolean.py @@ -3,9 +3,11 @@ from typing import Any, Dict, List import attrs +from typing_extensions import Self from .. import settings from ..schema import Field +from ..types import IDescriptor @attrs.define(kw_only=True, repr=False) @@ -59,6 +61,16 @@ def value_writer(cell: Any): # Metadata + @classmethod + def metadata_import( + cls, descriptor: IDescriptor, **options: Any + ) -> Self: + if "trueValues" in descriptor.keys(): + descriptor["trueValues"] = descriptor["trueValues"] + settings.DEFAULT_TRUE_VALUES + if "falseValues" in descriptor.keys(): + descriptor["falseValues"] = descriptor["falseValues"] + settings.DEFAULT_FALSE_VALUES + return super().metadata_import(descriptor, **options) + metadata_profile_patch = { "properties": { "trueValues": {"type": "array", "items": {"type": "string"}}, From e1c59658a7acea43299dac513356961e3af4442a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Fri, 5 Jan 2024 15:34:50 +0100 Subject: [PATCH 05/21] Refactoring --- frictionless/fields/__spec__/test_boolean.py | 30 +++++++------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/frictionless/fields/__spec__/test_boolean.py b/frictionless/fields/__spec__/test_boolean.py index 9d4f7a4863..4e3b9eb99e 100644 --- a/frictionless/fields/__spec__/test_boolean.py +++ b/frictionless/fields/__spec__/test_boolean.py @@ -45,25 +45,17 @@ def test_boolean_read_cell(format, source, target, options): assert cell == target -def test_boolean_from_schema_descriptor_read_cell(): - schema = Schema.from_descriptor( - { - "fields": [ - {"name": "IsTrue", "type": "boolean", "trueValues": ["yes"], "falseValues": ["no"]} - ] - } - ) - - source = "true" - target = True - - fields = schema.fields - cell, notes = fields[0].read_cell(source) - assert cell == target - - source = "false" - target = False - +@pytest.mark.parametrize( + "source, target, options", + [ + ("true", True, {"trueValues": ["yes"]}), + ("no", False, {"falseValues": ["no"]}) + ], +) +def test_boolean_from_schema_descriptor_read_cell(source, target, options): + schema_descriptor = {"fields": [{"name": "IsTrue", "type": "boolean"}]} + schema_descriptor["fields"][0].update(options) + schema = Schema.from_descriptor(schema_descriptor) fields = schema.fields cell, notes = fields[0].read_cell(source) assert cell == target From e77e4164d64aab28c362f7ae0f764cac6216e15d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Fri, 5 Jan 2024 15:47:40 +0100 Subject: [PATCH 06/21] Add many test cases for test boolean field from schema descriptor read cell: tests pass --- frictionless/fields/__spec__/test_boolean.py | 31 +++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/frictionless/fields/__spec__/test_boolean.py b/frictionless/fields/__spec__/test_boolean.py index 4e3b9eb99e..8cdfa0d532 100644 --- a/frictionless/fields/__spec__/test_boolean.py +++ b/frictionless/fields/__spec__/test_boolean.py @@ -48,8 +48,37 @@ def test_boolean_read_cell(format, source, target, options): @pytest.mark.parametrize( "source, target, options", [ + (True, True, {}), + ("true", True, {}), + ("True", True, {}), + ("TRUE", True, {}), + ("1", True, {}), + (True, True, {"trueValues": ["yes"]}), ("true", True, {"trueValues": ["yes"]}), - ("no", False, {"falseValues": ["no"]}) + ("True", True, {"trueValues": ["yes"]}), + ("TRUE", True, {"trueValues": ["yes"]}), + ("1", True, {"trueValues": ["yes"]}), + ("yes", True, {"trueValues": ["yes"]}), + (False, False, {}), + ("false", False, {}), + ("False", False, {}), + ("FALSE", False, {}), + ("0", False, {}), + (False, False, {"falseValues": ["no"]}), + ("false", False, {"falseValues": ["no"]}), + ("False", False, {"falseValues": ["no"]}), + ("FALSE", False, {"falseValues": ["no"]}), + ("0", False, {"falseValues": ["no"]}), + ("no", False, {"falseValues": ["no"]}), + ("yes", None, {}), + ("no", None, {}), + (0, None, {}), + (1, None, {}), + ("YES", None, {}), + ("NO", None, {}), + ("", None, {}), + ("Yes", None, {"trueValues": ["yes"]}), + ("No", None, {"falseValues": ["no"]}), ], ) def test_boolean_from_schema_descriptor_read_cell(source, target, options): From a93c03553b7d157f32b3fff9ca5a36493585cf38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Mon, 15 Jan 2024 17:27:51 +0100 Subject: [PATCH 07/21] Fix test cases to expected behavior --- frictionless/fields/__spec__/test_boolean.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/frictionless/fields/__spec__/test_boolean.py b/frictionless/fields/__spec__/test_boolean.py index 8cdfa0d532..b76f26a556 100644 --- a/frictionless/fields/__spec__/test_boolean.py +++ b/frictionless/fields/__spec__/test_boolean.py @@ -32,9 +32,9 @@ ("default", "3.14", None, {}), ("default", "", None, {}), ("default", "Yes", None, {"trueValues": ["yes"]}), - ("default", "true", True, {"trueValues": ["yes"]}), + ("default", "true", None, {"trueValues": ["yes"]}), ("default", "No", None, {"falseValues": ["no"]}), - ("default", "false", False, {"falseValues": ["no"]}), + ("default", "false", None, {"falseValues": ["no"]}), ], ) def test_boolean_read_cell(format, source, target, options): @@ -54,10 +54,6 @@ def test_boolean_read_cell(format, source, target, options): ("TRUE", True, {}), ("1", True, {}), (True, True, {"trueValues": ["yes"]}), - ("true", True, {"trueValues": ["yes"]}), - ("True", True, {"trueValues": ["yes"]}), - ("TRUE", True, {"trueValues": ["yes"]}), - ("1", True, {"trueValues": ["yes"]}), ("yes", True, {"trueValues": ["yes"]}), (False, False, {}), ("false", False, {}), @@ -65,10 +61,6 @@ def test_boolean_read_cell(format, source, target, options): ("FALSE", False, {}), ("0", False, {}), (False, False, {"falseValues": ["no"]}), - ("false", False, {"falseValues": ["no"]}), - ("False", False, {"falseValues": ["no"]}), - ("FALSE", False, {"falseValues": ["no"]}), - ("0", False, {"falseValues": ["no"]}), ("no", False, {"falseValues": ["no"]}), ("yes", None, {}), ("no", None, {}), @@ -78,7 +70,15 @@ def test_boolean_read_cell(format, source, target, options): ("NO", None, {}), ("", None, {}), ("Yes", None, {"trueValues": ["yes"]}), + ("true", None, {"trueValues": ["yes"]}), + ("True", None, {"trueValues": ["yes"]}), + ("TRUE", None, {"trueValues": ["yes"]}), + ("1", None, {"trueValues": ["yes"]}), ("No", None, {"falseValues": ["no"]}), + ("false", None, {"falseValues": ["no"]}), + ("False", None, {"falseValues": ["no"]}), + ("FALSE", None, {"falseValues": ["no"]}), + ("0", None, {"falseValues": ["no"]}), ], ) def test_boolean_from_schema_descriptor_read_cell(source, target, options): From c539d671cd4cbb8cfee42d1111992492039a91f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Mon, 15 Jan 2024 17:29:10 +0100 Subject: [PATCH 08/21] Fix failed tests --- frictionless/fields/boolean.py | 10 ---------- frictionless/schema/field.py | 4 ++-- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/frictionless/fields/boolean.py b/frictionless/fields/boolean.py index 6e2ca211fe..1c42060c60 100644 --- a/frictionless/fields/boolean.py +++ b/frictionless/fields/boolean.py @@ -61,16 +61,6 @@ def value_writer(cell: Any): # Metadata - @classmethod - def metadata_import( - cls, descriptor: IDescriptor, **options: Any - ) -> Self: - if "trueValues" in descriptor.keys(): - descriptor["trueValues"] = descriptor["trueValues"] + settings.DEFAULT_TRUE_VALUES - if "falseValues" in descriptor.keys(): - descriptor["falseValues"] = descriptor["falseValues"] + settings.DEFAULT_FALSE_VALUES - return super().metadata_import(descriptor, **options) - metadata_profile_patch = { "properties": { "trueValues": {"type": "array", "items": {"type": "string"}}, diff --git a/frictionless/schema/field.py b/frictionless/schema/field.py index e1b13bf1ff..bde8efec14 100644 --- a/frictionless/schema/field.py +++ b/frictionless/schema/field.py @@ -205,10 +205,10 @@ def from_descriptor( # type: ignore if isinstance(descriptor, dict): if "trueValues" in descriptor.keys(): assert descriptor["type"] == "boolean" - descriptor["trueValues"] = descriptor["trueValues"] + settings.DEFAULT_TRUE_VALUES + descriptor["trueValues"] = descriptor["trueValues"] if "falseValues" in descriptor.keys(): assert descriptor["type"] == "boolean" - descriptor["falseValues"] = descriptor["falseValues"] + settings.DEFAULT_FALSE_VALUES + descriptor["falseValues"] = descriptor["falseValues"] return super().from_descriptor(descriptor) # Metadata From 7107b8d23506fc82308ee1c423df7c33d8a020e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Mon, 15 Jan 2024 17:31:14 +0100 Subject: [PATCH 09/21] TDD: add test case to fix example value in boolean field with customized trueValues or falseValues properties in schema descriptor --- frictionless/fields/__spec__/test_boolean.py | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/frictionless/fields/__spec__/test_boolean.py b/frictionless/fields/__spec__/test_boolean.py index b76f26a556..0afc62a402 100644 --- a/frictionless/fields/__spec__/test_boolean.py +++ b/frictionless/fields/__spec__/test_boolean.py @@ -1,6 +1,7 @@ import pytest from frictionless import Field, Schema +from frictionless.resources.table import TableResource # General @@ -88,3 +89,25 @@ def test_boolean_from_schema_descriptor_read_cell(source, target, options): fields = schema.fields cell, notes = fields[0].read_cell(source) assert cell == target + + +def test_boolean_from_schema_descriptor_with_example_fix_issue_1610(): + schema_descriptor = { + "$schema": "https://frictionlessdata.io/schemas/table-schema.json", + "fields": [ + { + "name": "IsTrue", + "type": "boolean", + "trueValues": ["yes"], + "falseValues": ["no"], + "example": "no" + } + ] + } + + schema = Schema.from_descriptor(schema_descriptor) + fields = schema.fields + source = "yes" + cell, notes = fields[0].read_cell(source) + assert cell + \ No newline at end of file From 3c7cd548202c3205ef0a24466b67e90976f54af8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Tue, 16 Jan 2024 10:44:32 +0100 Subject: [PATCH 10/21] TDD: test passes: edit 'field.true_values' and 'field.false_values' to validate 'example' value from descriptor boolean field --- frictionless/schema/field.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frictionless/schema/field.py b/frictionless/schema/field.py index bde8efec14..1065967642 100644 --- a/frictionless/schema/field.py +++ b/frictionless/schema/field.py @@ -281,6 +281,12 @@ def metadata_validate(cls, descriptor: IDescriptor): # type: ignore name=descriptor.get("name", "example"), format=descriptor.get("format", "default"), # type: ignore ) + if type == "boolean": + # 'example' value must be compared to customized 'trueValues' and 'falseValues' + if 'trueValues' in descriptor.keys(): + field.true_values = descriptor['trueValues'] + if 'falseValues' in descriptor.keys(): + field.false_values = descriptor['falseValues'] _, notes = field.read_cell(example) if notes is not None: note = f'example value "{example}" for field "{field.name}" is not valid' From e26d53395e38c8f9d0071182856a81a262ed9def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Tue, 16 Jan 2024 10:45:00 +0100 Subject: [PATCH 11/21] Refacto: sort and remove useless imports --- frictionless/fields/boolean.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/frictionless/fields/boolean.py b/frictionless/fields/boolean.py index 1c42060c60..6d78984fe1 100644 --- a/frictionless/fields/boolean.py +++ b/frictionless/fields/boolean.py @@ -3,11 +3,9 @@ from typing import Any, Dict, List import attrs -from typing_extensions import Self from .. import settings from ..schema import Field -from ..types import IDescriptor @attrs.define(kw_only=True, repr=False) From 8e7c1f94ddcf609793bec1f60af80440b4e93756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Tue, 16 Jan 2024 10:57:05 +0100 Subject: [PATCH 12/21] Refacto: factorize test cases --- frictionless/fields/__spec__/test_boolean.py | 52 ++++---------------- 1 file changed, 9 insertions(+), 43 deletions(-) diff --git a/frictionless/fields/__spec__/test_boolean.py b/frictionless/fields/__spec__/test_boolean.py index 0afc62a402..b898c077c5 100644 --- a/frictionless/fields/__spec__/test_boolean.py +++ b/frictionless/fields/__spec__/test_boolean.py @@ -1,7 +1,6 @@ import pytest from frictionless import Field, Schema -from frictionless.resources.table import TableResource # General @@ -34,60 +33,28 @@ ("default", "", None, {}), ("default", "Yes", None, {"trueValues": ["yes"]}), ("default", "true", None, {"trueValues": ["yes"]}), + ("default", "True", None, {"trueValues": ["yes"]}), + ("default", "TRUE", None, {"trueValues": ["yes"]}), + ("default", "1", None, {"trueValues": ["yes"]}), ("default", "No", None, {"falseValues": ["no"]}), ("default", "false", None, {"falseValues": ["no"]}), + ("default", "False", None, {"falseValues": ["no"]}), + ("default", "FALSE", None, {"falseValues": ["no"]}), + ("default", "0", None, {"falseValues": ["no"]}), ], ) def test_boolean_read_cell(format, source, target, options): descriptor = {"name": "name", "type": "boolean", "format": format} descriptor.update(options) field = Field.from_descriptor(descriptor) - cell, notes = field.read_cell(source) + cell, _ = field.read_cell(source) assert cell == target - -@pytest.mark.parametrize( - "source, target, options", - [ - (True, True, {}), - ("true", True, {}), - ("True", True, {}), - ("TRUE", True, {}), - ("1", True, {}), - (True, True, {"trueValues": ["yes"]}), - ("yes", True, {"trueValues": ["yes"]}), - (False, False, {}), - ("false", False, {}), - ("False", False, {}), - ("FALSE", False, {}), - ("0", False, {}), - (False, False, {"falseValues": ["no"]}), - ("no", False, {"falseValues": ["no"]}), - ("yes", None, {}), - ("no", None, {}), - (0, None, {}), - (1, None, {}), - ("YES", None, {}), - ("NO", None, {}), - ("", None, {}), - ("Yes", None, {"trueValues": ["yes"]}), - ("true", None, {"trueValues": ["yes"]}), - ("True", None, {"trueValues": ["yes"]}), - ("TRUE", None, {"trueValues": ["yes"]}), - ("1", None, {"trueValues": ["yes"]}), - ("No", None, {"falseValues": ["no"]}), - ("false", None, {"falseValues": ["no"]}), - ("False", None, {"falseValues": ["no"]}), - ("FALSE", None, {"falseValues": ["no"]}), - ("0", None, {"falseValues": ["no"]}), - ], -) -def test_boolean_from_schema_descriptor_read_cell(source, target, options): schema_descriptor = {"fields": [{"name": "IsTrue", "type": "boolean"}]} schema_descriptor["fields"][0].update(options) schema = Schema.from_descriptor(schema_descriptor) fields = schema.fields - cell, notes = fields[0].read_cell(source) + cell, _ = fields[0].read_cell(source) assert cell == target @@ -108,6 +75,5 @@ def test_boolean_from_schema_descriptor_with_example_fix_issue_1610(): schema = Schema.from_descriptor(schema_descriptor) fields = schema.fields source = "yes" - cell, notes = fields[0].read_cell(source) + cell, _ = fields[0].read_cell(source) assert cell - \ No newline at end of file From 0e0aa0e20a26019ec62438c07f03c439e7672598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Tue, 16 Jan 2024 11:14:40 +0100 Subject: [PATCH 13/21] Refacto and add test cases corresponding to issue 1610 --- frictionless/fields/__spec__/test_boolean.py | 29 +++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/frictionless/fields/__spec__/test_boolean.py b/frictionless/fields/__spec__/test_boolean.py index b898c077c5..3fad396018 100644 --- a/frictionless/fields/__spec__/test_boolean.py +++ b/frictionless/fields/__spec__/test_boolean.py @@ -58,22 +58,25 @@ def test_boolean_read_cell(format, source, target, options): assert cell == target -def test_boolean_from_schema_descriptor_with_example_fix_issue_1610(): +@pytest.mark.parametrize( + "source, target, options", + [ + (True, True, {'trueValues': ["yes"], "example": "yes"}), + ("yes", True, {'trueValues': ["yes"], "example": "yes"}), + ("true", None, {'trueValues': ["yes"], "example": "yes"}), + ("no", False, {'falseValues': ["no"], "example": "no"}), + ("no", False, {'falseValues': ["no"], "example": "no"}), + ("false", None, {'falseValues': ["no"], "example": "no"}), + ] +) +def test_boolean_from_schema_descriptor_with_valid_example_fix_issue_1610( + source, target, options): schema_descriptor = { "$schema": "https://frictionlessdata.io/schemas/table-schema.json", - "fields": [ - { - "name": "IsTrue", - "type": "boolean", - "trueValues": ["yes"], - "falseValues": ["no"], - "example": "no" - } - ] + "fields": [{"name": "IsTrue", "type": "boolean"}] } - + schema_descriptor["fields"][0].update(options) schema = Schema.from_descriptor(schema_descriptor) fields = schema.fields - source = "yes" cell, _ = fields[0].read_cell(source) - assert cell + assert cell == target From 151c81f8781480135e1d35b133aa2be39f5dd9e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Tue, 16 Jan 2024 11:42:44 +0100 Subject: [PATCH 14/21] Add test case for invalid value example in schema descriptor to fix issue 1610 --- frictionless/fields/__spec__/test_boolean.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/frictionless/fields/__spec__/test_boolean.py b/frictionless/fields/__spec__/test_boolean.py index 3fad396018..33c7a1644f 100644 --- a/frictionless/fields/__spec__/test_boolean.py +++ b/frictionless/fields/__spec__/test_boolean.py @@ -1,7 +1,8 @@ import pytest from frictionless import Field, Schema - +from frictionless.errors.metadata import SchemaError +from frictionless.exception import FrictionlessException # General @@ -80,3 +81,15 @@ def test_boolean_from_schema_descriptor_with_valid_example_fix_issue_1610( fields = schema.fields cell, _ = fields[0].read_cell(source) assert cell == target + + +def test_boolean_from_schema_descriptor_with_invalid_example_fix_issue_1610(): + schema_descriptor = { + "$schema": "https://frictionlessdata.io/schemas/table-schema.json", + "fields": [{"name": "IsTrue", "type": "boolean", + 'falseValues': ["no"], "example": "unvalid"}] + } + with pytest.raises(FrictionlessException) as excinfo: + schema = Schema.from_descriptor(schema_descriptor) + err = excinfo.value.error + assert isinstance(err, SchemaError) From 9e08f2d7c9beb1bfebb113c1a20b87f3ea737f82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Tue, 16 Jan 2024 12:20:20 +0100 Subject: [PATCH 15/21] Refacto: reformat file --- frictionless/schema/field.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frictionless/schema/field.py b/frictionless/schema/field.py index 1065967642..b27484bac5 100644 --- a/frictionless/schema/field.py +++ b/frictionless/schema/field.py @@ -283,10 +283,10 @@ def metadata_validate(cls, descriptor: IDescriptor): # type: ignore ) if type == "boolean": # 'example' value must be compared to customized 'trueValues' and 'falseValues' - if 'trueValues' in descriptor.keys(): - field.true_values = descriptor['trueValues'] - if 'falseValues' in descriptor.keys(): - field.false_values = descriptor['falseValues'] + if "trueValues" in descriptor.keys(): + field.true_values = descriptor["trueValues"] + if "falseValues" in descriptor.keys(): + field.false_values = descriptor["falseValues"] _, notes = field.read_cell(example) if notes is not None: note = f'example value "{example}" for field "{field.name}" is not valid' From cdea8026e2a79ebefe6cfd296d75a876e2fc2576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Thu, 25 Jan 2024 09:50:12 +0100 Subject: [PATCH 16/21] Linting: lint and format files --- frictionless/fields/__spec__/test_boolean.py | 32 ++++++++++++-------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/frictionless/fields/__spec__/test_boolean.py b/frictionless/fields/__spec__/test_boolean.py index 33c7a1644f..7924c56295 100644 --- a/frictionless/fields/__spec__/test_boolean.py +++ b/frictionless/fields/__spec__/test_boolean.py @@ -3,6 +3,7 @@ from frictionless import Field, Schema from frictionless.errors.metadata import SchemaError from frictionless.exception import FrictionlessException + # General @@ -62,19 +63,20 @@ def test_boolean_read_cell(format, source, target, options): @pytest.mark.parametrize( "source, target, options", [ - (True, True, {'trueValues': ["yes"], "example": "yes"}), - ("yes", True, {'trueValues': ["yes"], "example": "yes"}), - ("true", None, {'trueValues': ["yes"], "example": "yes"}), - ("no", False, {'falseValues': ["no"], "example": "no"}), - ("no", False, {'falseValues': ["no"], "example": "no"}), - ("false", None, {'falseValues': ["no"], "example": "no"}), - ] + (True, True, {"trueValues": ["yes"], "example": "yes"}), + ("yes", True, {"trueValues": ["yes"], "example": "yes"}), + ("true", None, {"trueValues": ["yes"], "example": "yes"}), + ("no", False, {"falseValues": ["no"], "example": "no"}), + ("no", False, {"falseValues": ["no"], "example": "no"}), + ("false", None, {"falseValues": ["no"], "example": "no"}), + ], ) def test_boolean_from_schema_descriptor_with_valid_example_fix_issue_1610( - source, target, options): + source, target, options +): schema_descriptor = { "$schema": "https://frictionlessdata.io/schemas/table-schema.json", - "fields": [{"name": "IsTrue", "type": "boolean"}] + "fields": [{"name": "IsTrue", "type": "boolean"}], } schema_descriptor["fields"][0].update(options) schema = Schema.from_descriptor(schema_descriptor) @@ -86,10 +88,16 @@ def test_boolean_from_schema_descriptor_with_valid_example_fix_issue_1610( def test_boolean_from_schema_descriptor_with_invalid_example_fix_issue_1610(): schema_descriptor = { "$schema": "https://frictionlessdata.io/schemas/table-schema.json", - "fields": [{"name": "IsTrue", "type": "boolean", - 'falseValues': ["no"], "example": "unvalid"}] + "fields": [ + { + "name": "IsTrue", + "type": "boolean", + "falseValues": ["no"], + "example": "unvalid", + } + ], } with pytest.raises(FrictionlessException) as excinfo: - schema = Schema.from_descriptor(schema_descriptor) + Schema.from_descriptor(schema_descriptor) err = excinfo.value.error assert isinstance(err, SchemaError) From 07eefdcc24c67f6257f708e7f4b2146be84cbc19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Mon, 29 Jan 2024 11:54:50 +0100 Subject: [PATCH 17/21] Refacto: rmove useless lines --- frictionless/schema/field.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/frictionless/schema/field.py b/frictionless/schema/field.py index b27484bac5..b0df412204 100644 --- a/frictionless/schema/field.py +++ b/frictionless/schema/field.py @@ -204,10 +204,8 @@ def from_descriptor( # type: ignore ) -> Self: if isinstance(descriptor, dict): if "trueValues" in descriptor.keys(): - assert descriptor["type"] == "boolean" descriptor["trueValues"] = descriptor["trueValues"] if "falseValues" in descriptor.keys(): - assert descriptor["type"] == "boolean" descriptor["falseValues"] = descriptor["falseValues"] return super().from_descriptor(descriptor) From c2b8fafe4a7a02cc519bfcadcc8cb6e75aa21b85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Am=C3=A9lie=20Rondot?= Date: Mon, 29 Jan 2024 12:40:20 +0100 Subject: [PATCH 18/21] Refacto: remove useless lines 2 --- frictionless/schema/field.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/frictionless/schema/field.py b/frictionless/schema/field.py index b0df412204..4e0569aa2f 100644 --- a/frictionless/schema/field.py +++ b/frictionless/schema/field.py @@ -3,11 +3,9 @@ import decimal import re from functools import partial -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, Optional -from typing import Pattern, Union +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, Optional, Pattern import attrs -from typing_extensions import Self from .. import errors, settings from ..exception import FrictionlessException @@ -196,19 +194,6 @@ def value_writer(cell: Any): return value_writer - @classmethod - def from_descriptor( # type: ignore - cls, - descriptor: Union[IDescriptor, str], - **options: Any, - ) -> Self: - if isinstance(descriptor, dict): - if "trueValues" in descriptor.keys(): - descriptor["trueValues"] = descriptor["trueValues"] - if "falseValues" in descriptor.keys(): - descriptor["falseValues"] = descriptor["falseValues"] - return super().from_descriptor(descriptor) - # Metadata metadata_type = "field" From a7a01b4f685ce5eb3c88bd56b1f7d3e0a2fb99f2 Mon Sep 17 00:00:00 2001 From: Pierre Camilleri <22995923+pierrecamilleri@users.noreply.github.com> Date: Tue, 23 Apr 2024 17:11:18 +0200 Subject: [PATCH 19/21] =?UTF-8?q?=F0=9F=94=B5=20typo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frictionless/fields/__spec__/test_boolean.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frictionless/fields/__spec__/test_boolean.py b/frictionless/fields/__spec__/test_boolean.py index 7924c56295..574bf81c2c 100644 --- a/frictionless/fields/__spec__/test_boolean.py +++ b/frictionless/fields/__spec__/test_boolean.py @@ -66,7 +66,7 @@ def test_boolean_read_cell(format, source, target, options): (True, True, {"trueValues": ["yes"], "example": "yes"}), ("yes", True, {"trueValues": ["yes"], "example": "yes"}), ("true", None, {"trueValues": ["yes"], "example": "yes"}), - ("no", False, {"falseValues": ["no"], "example": "no"}), + (False, False, {"falseValues": ["no"], "example": "no"}), ("no", False, {"falseValues": ["no"], "example": "no"}), ("false", None, {"falseValues": ["no"], "example": "no"}), ], @@ -93,7 +93,7 @@ def test_boolean_from_schema_descriptor_with_invalid_example_fix_issue_1610(): "name": "IsTrue", "type": "boolean", "falseValues": ["no"], - "example": "unvalid", + "example": "invalid", } ], } From 42b14fa73c4dfaa8fe47313930aa95c08ab52fea Mon Sep 17 00:00:00 2001 From: Pierre Camilleri <22995923+pierrecamilleri@users.noreply.github.com> Date: Fri, 3 May 2024 10:25:14 +0200 Subject: [PATCH 20/21] fix: from_descriptor not possible `from_descriptor` leads to a `RecursionError` --- frictionless/schema/field.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/frictionless/schema/field.py b/frictionless/schema/field.py index 4e0569aa2f..9ccc5f97e7 100644 --- a/frictionless/schema/field.py +++ b/frictionless/schema/field.py @@ -3,7 +3,8 @@ import decimal import re from functools import partial -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, Optional, Pattern +from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, + Optional, Pattern) import attrs @@ -50,7 +51,9 @@ class Field(Metadata): For example: "default","array" etc. """ - missing_values: List[str] = attrs.field(factory=settings.DEFAULT_MISSING_VALUES.copy) + missing_values: List[str] = attrs.field( + factory=settings.DEFAULT_MISSING_VALUES.copy + ) """ List of string values to be set as missing values in the field. If any of string in missing values is found in the field value then it is set as None. @@ -260,10 +263,12 @@ def metadata_validate(cls, descriptor: IDescriptor): # type: ignore if example: type = descriptor.get("type") Class = system.select_field_class(type) + field = Class( - name=descriptor.get("name", "example"), - format=descriptor.get("format", "default"), # type: ignore + name=descriptor.get("name"), # type: ignore + format=descriptor.get("format", "default"), ) + if type == "boolean": # 'example' value must be compared to customized 'trueValues' and 'falseValues' if "trueValues" in descriptor.keys(): @@ -272,7 +277,9 @@ def metadata_validate(cls, descriptor: IDescriptor): # type: ignore field.false_values = descriptor["falseValues"] _, notes = field.read_cell(example) if notes is not None: - note = f'example value "{example}" for field "{field.name}" is not valid' + note = ( + f'example value "{example}" for field "{field.name}" is not valid' + ) yield errors.FieldError(note=note) # Misleading From f127067f1cca8962201acec7bac10d29fd327c01 Mon Sep 17 00:00:00 2001 From: Pierre Camilleri <22995923+pierrecamilleri@users.noreply.github.com> Date: Fri, 30 Aug 2024 13:20:06 +0200 Subject: [PATCH 21/21] fix: lint --- frictionless/schema/field.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frictionless/schema/field.py b/frictionless/schema/field.py index 9ccc5f97e7..5b9f8c3eb5 100644 --- a/frictionless/schema/field.py +++ b/frictionless/schema/field.py @@ -3,8 +3,7 @@ import decimal import re from functools import partial -from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, - Optional, Pattern) +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, List, Optional, Pattern import attrs