From 7eed13b5f8af42c48a18d95bfaadc7adf93f5cd5 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Mon, 2 Nov 2026 04:34:06 +0530 Subject: [PATCH] fix: let int() decide what an int is in handle_arg_string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `handle_arg_string` used `str.isnumeric()` to decide whether a `--data_args` value was an integer. That predicate disagrees with `int()` in both directions: - `"-5".isnumeric()` is False, so a negative integer skipped the int branch and came back from `float()` as `-5.0`. `simple_parse_args_string` feeds straight into `load_dataset(**kwargs)`, so `--data_args num_proc=-1` silently handed a float to a parameter that wants an int. No error, just the wrong type. - `"½".isnumeric()` is True but `int("½")` raises, so a Unicode numeral raised an unhandled ValueError. The same applies to `"²"` and to Roman numerals such as `"Ⅷ"`. At the call site in `__main__.py` that ValueError is caught by the `load_from_disk` handler and resurfaces as a dataset error, which points at the wrong thing. Trying `int()` and falling through on ValueError removes both. Values that already parsed keep parsing the same way, including fullwidth digits. Adds tests/test_utils.py covering signed integers, floats, Unicode numerals, booleans, plain strings, and the end-to-end `simple_parse_args_string` path. --- sparsify/utils.py | 6 +++- tests/test_utils.py | 72 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 tests/test_utils.py diff --git a/sparsify/utils.py b/sparsify/utils.py index 1877005c..f67cdda1 100644 --- a/sparsify/utils.py +++ b/sparsify/utils.py @@ -108,8 +108,12 @@ def handle_arg_string(arg): return True elif arg.lower() == "false": return False - elif arg.isnumeric(): + # int() decides, not str.isnumeric(): "-5" is not numeric but is an int, and + # "½" is numeric but int() rejects it. + try: return int(arg) + except ValueError: + pass try: return float(arg) except ValueError: diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..f451327c --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,72 @@ +"""`--data_args` values must keep the type the user wrote. + +`handle_arg_string` used `str.isnumeric()` to decide whether a value was an +integer. That predicate disagrees with `int()` in both directions: "-5" is not +numeric but is an integer, and "½" is numeric but `int()` rejects it. So a +negative integer silently arrived as a float, and a Unicode numeral raised an +unhandled ValueError on the way to `load_dataset`. +""" + +import pytest + +from sparsify.utils import handle_arg_string, simple_parse_args_string + + +@pytest.mark.parametrize( + "text,expected", + [ + ("5", 5), + ("0", 0), + ("-5", -5), + ("+5", 5), + ("-0", 0), + ], +) +def test_integers_stay_integers(text, expected): + """Negative and signed integers used to come back as floats.""" + result = handle_arg_string(text) + assert result == expected + assert isinstance(result, int) + + +@pytest.mark.parametrize("text", ["3.5", "-3.5", "1e3"]) +def test_floats_are_still_floats(text): + result = handle_arg_string(text) + assert isinstance(result, float) + assert result == float(text) + + +@pytest.mark.parametrize("text", ["½", "²", "Ⅷ"]) +def test_unicode_numerals_do_not_raise(text): + """str.isnumeric() accepts these; int() does not, so this used to crash.""" + assert handle_arg_string(text) == text + + +@pytest.mark.parametrize( + "text,expected", + [("true", True), ("True", True), ("false", False), ("FALSE", False)], +) +def test_booleans(text, expected): + assert handle_arg_string(text) is expected + + +def test_plain_strings_pass_through(): + assert handle_arg_string("train") == "train" + + +def test_parse_args_string_keeps_integer_types(): + """The end-to-end path a `--data_args` value takes.""" + parsed = simple_parse_args_string("num_proc=-1,name=wikitext,ratio=0.5,flag=true") + + assert parsed == { + "num_proc": -1, + "name": "wikitext", + "ratio": 0.5, + "flag": True, + } + assert isinstance(parsed["num_proc"], int) + + +def test_parse_args_string_empty(): + assert simple_parse_args_string("") == {} + assert simple_parse_args_string(" ") == {}