diff --git a/README.md b/README.md index e11b321..c7cb3ca 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,12 @@ part of your geospatial project. # Version Changes +## 3.1.7 +### Bug fix + - Warn (or raise the new `DbfNumericDataLoss` in strict mode) when a number is too wide for its + "N" or "F" field and has to be truncated, instead of silently writing a different number. + `DbfStringDataLoss` and `DbfNumericDataLoss` now share a `DbfDataLoss` base class. + ## 3.1.6 ### Feature - Encodings can now be read from .cpg files (and optionally written to them). @@ -1073,7 +1079,9 @@ Numeric fields are created using the 'N' type (or the 'F' type, which is exactly By default the fourth decimal argument is set to zero, essentially creating an integer field. To store floats you must set the decimal argument to the precision of your choice. To store very large numbers you must increase the field length size to the total number of digits -(including comma and minus). +(including comma and minus). A number that is wider than its field has to be truncated to keep the +record layout intact, which writes a different number, so PyShp warns about this (and raises +`DbfNumericDataLoss` in strict mode). >>> w = shapefile.Writer('tests/shapefiles/test/dtype') diff --git a/changelog.txt b/changelog.txt index fa92b2f..4c651b4 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,7 @@ +VERSION 3.1.7 +2026-08-06 + * Warn (or raise the new DbfNumericDataLoss in strict mode) when a number is too wide for its "N" or "F" field and has to be truncated, instead of silently writing a different number. + VERSION 3.1.6 2026-07-25 * Encodings can now be read from .cpg files (and optionally written to them). diff --git a/src/shapefile.py b/src/shapefile.py index 68aceb1..114e51c 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -239,7 +239,15 @@ class PossibleDataLoss(Warning): pass -class DbfStringDataLoss(ValueError): +class DbfDataLoss(ValueError): + pass + + +class DbfStringDataLoss(DbfDataLoss): + pass + + +class DbfNumericDataLoss(DbfDataLoss): pass @@ -457,6 +465,35 @@ def check_and_trim(decoded_pad_bytes: dict[str, bytes]) -> None: return padded, trimmed +def _pack_dbf_number( + formatted: str, + size: int, + value: RecordValue, + field_name: str, + strict: bool = True, +) -> str: + """Right justifies an already formatted number in a field of width size. + + A truncated number is a different number, not a lossy version of the data + like a truncated string is, so warn about it (or raise in strict mode), + as the "C" and "M" field paths do. + """ + if len(formatted) > size: + msg = ( + f"Formatted: {formatted} of data: {value!r} needs {len(formatted)} bytes, " + f"and was truncated to: {formatted[:size]} " + f"to fit within the {size=} bytes of field: {field_name!r}, " + "changing the number that is written. " + "To avoid data loss, make a new Writer or dbfWriter and call .field " + "with a bigger size (and if applicable, a smaller decimal). " + ) + if strict: + raise DbfNumericDataLoss(msg) + warnings.warn(msg, category=PossibleDataLoss) + + return formatted[:size].rjust(size) + + def _try_to_decode_dbf_name_or_text_field( b: bytes, pad_bytes: bytes, # Pad bytes will be trimmed from the RHS (end) of b. @@ -4451,14 +4488,14 @@ def _record(self, record: list[RecordValue]) -> None: except ValueError: # forcing directly to int failed, so was probably a float. num_val = int(float(cast(float, value))) - str_val = format(num_val, "d")[:size].rjust( - size - ) # caps the size if exceeds the field size + str_val = _pack_dbf_number( + format(num_val, "d"), size, value, fieldName, self.strict + ) else: f_val = float(cast(float, value)) - str_val = format(f_val, f".{deci}f")[:size].rjust( - size - ) # caps the size if exceeds the field size + str_val = _pack_dbf_number( + format(f_val, f".{deci}f"), size, value, fieldName, self.strict + ) elif fieldType == "D": # date: 8 bytes - date stored as a string in the format YYYYMMDD. if isinstance(value, list) and len(value) == 3: diff --git a/tests/hypothesis_tests.py b/tests/hypothesis_tests.py index d5157a7..16c0d08 100644 --- a/tests/hypothesis_tests.py +++ b/tests/hypothesis_tests.py @@ -881,7 +881,9 @@ def _write_fields_and_records_to_strict(w, fields, records): ] try: w.record(*rec_list) - except shp.DbfStringDataLoss: + except shp.DbfDataLoss: + # Numbers too wide for their field are rejected in strict mode too, + # e.g. round(9999999.96875, 1) needs 10 chars in an N field of size 9. written_records.append(None) else: written_records.append(rec_list) diff --git a/tests/test_shapefile.py b/tests/test_shapefile.py index 168bd63..3752349 100644 --- a/tests/test_shapefile.py +++ b/tests/test_shapefile.py @@ -9,6 +9,7 @@ import os.path from pathlib import Path import shutil +import warnings # third party imports import pytest @@ -2068,6 +2069,100 @@ def test_encode_dbf_field_padding_bytes_errors(value,encoded_len,codec,errors): w.record(value) w.close() + +def _write_one_field_record(field_type, size, decimal, value, strict): + """Returns the raw bytes of the single field, and the value read back.""" + stream = io.BytesIO() + w = shapefile.DbfWriter(dbf=stream, strict=strict) + w.field("V", field_type, size=size, decimal=decimal) + w.record(value) + w.close() + + raw = stream.getvalue() + # 32 byte header + 32 bytes per field + terminator + the record's deletion flag. + start = 32 + 32 + 1 + 1 + with shapefile.DbfReader(dbf=io.BytesIO(raw)) as r: + # DeletionFlag is r.fields[0]. + written_size = r.fields[1][2] + return raw[start:start + written_size], r.record(0)[0] + +# Numbers that do not fit their field, with the corrupted bytes and the +# different number that PyShp writes in their place. +NUMERIC_FIELD_OVERFLOWS = [ + ("N", 5, 0, 123456789, b"12345", 12345), + ("N", 5, 0, -123456, b"-1234", -1234), + ("N", 2, 0, -999, b"-9", -9), + ("N", 1, 0, -5, b"-", None), # nothing but the sign is left + ("N", 3, 0, 10 ** 30, b"100", 100), + ("N", 6, 2, 12345.67, b"12345.", 12345.0), # ends in a bare decimal point + ("N", 5, 2, 12345.67, b"12345", 12345.0), + ("N", 3, 1, -1.25, b"-1.", -1.0), + ("N", 9, 1, 9999999.96875, b"10000000.", 10000000.0), # rounding carries a digit + ("F", 5, 0, 123456789, b"12345", 12345), + ("F", 6, 2, 12345.67, b"12345.", 12345.0), + ("F", 4, 2, -1.25, b"-1.2", -1.2), +] + +# The same fields and values, sized so that nothing has to be truncated. +NUMERIC_FIELDS_THAT_FIT = [ + ("N", 5, 0, 12345), + ("N", 5, 0, -1234), + ("N", 1, 0, 7), + ("N", 8, 2, 12345.67), + ("N", 10, 3, 1.5), + ("F", 9, 1, 9999999.9), + ("F", 6, 2, -1.25), +] + +@pytest.mark.parametrize("field_type,size,decimal,value,expected_bytes,read_back", NUMERIC_FIELD_OVERFLOWS) +def test_numeric_field_overflow_warns_and_corrupts_the_value( + field_type, size, decimal, value, expected_bytes, read_back + ): + with pytest.warns(shapefile.PossibleDataLoss): + written, actual = _write_one_field_record(field_type, size, decimal, value, strict=False) + + assert written == expected_bytes + assert actual == read_back + assert actual != value + +@pytest.mark.parametrize("field_type,size,decimal,value,expected_bytes,read_back", NUMERIC_FIELD_OVERFLOWS) +def test_numeric_field_overflow_raises_in_strict_mode( + field_type, size, decimal, value, expected_bytes, read_back + ): + with pytest.raises(shapefile.DbfNumericDataLoss): + _write_one_field_record(field_type, size, decimal, value, strict=True) + +@pytest.mark.parametrize("strict", [False, True]) +@pytest.mark.parametrize("field_type,size,decimal,value", NUMERIC_FIELDS_THAT_FIT) +def test_numeric_field_that_fits_round_trips_without_warning(field_type, size, decimal, value, strict): + with warnings.catch_warnings(): + warnings.simplefilter("error", shapefile.PossibleDataLoss) + written, actual = _write_one_field_record(field_type, size, decimal, value, strict) + + assert len(written) == size + assert actual == value + +@pytest.mark.parametrize("strict", [False, True]) +@pytest.mark.parametrize("field_type,size,value,expected_bytes", [ + ("D", 4, datetime.date(2026, 8, 6), b"20260806"), + ("D", 20, datetime.date(2026, 8, 6), b"20260806"), + ("L", 4, True, b"T"), + ("L", 20, False, b"F"), +]) +def test_date_and_logical_fields_cannot_overflow(field_type, size, value, expected_bytes, strict): + # Field.from_unchecked forces "D" to 8 bytes and "L" to 1, so unlike the + # "N" and "F" fields, a mis-sized field cannot truncate the value. + with warnings.catch_warnings(): + warnings.simplefilter("error", shapefile.PossibleDataLoss) + written, _actual = _write_one_field_record(field_type, size, 0, value, strict) + + assert written == expected_bytes + +def test_dbf_data_loss_exceptions_share_a_base_class(): + assert issubclass(shapefile.DbfStringDataLoss, shapefile.DbfDataLoss) + assert issubclass(shapefile.DbfNumericDataLoss, shapefile.DbfDataLoss) + assert issubclass(shapefile.DbfDataLoss, ValueError) + LONG_FIELD_NAMES = [ ("ÀÀÀÀ०", 8, "utf-8", "strict"), # Encoded bytes are corrupted if truncated to 10 bytes ]