From 4f74a461e96655786d4e6b5a66e079aa55862c28 Mon Sep 17 00:00:00 2001 From: Ryan Duguid <152749594+ryanduguid@users.noreply.github.com> Date: Thu, 13 Aug 2026 02:50:13 +1000 Subject: [PATCH 1/4] fix: serialize epoch datetimes on Windows --- tests/test_api_client/test_serializer.py | 1 + xero_python/api_client/serializer.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_api_client/test_serializer.py b/tests/test_api_client/test_serializer.py index b190bf15..61ce9ab4 100644 --- a/tests/test_api_client/test_serializer.py +++ b/tests/test_api_client/test_serializer.py @@ -306,6 +306,7 @@ def test_serialize_datetime(value, expected): "value,expected", [ (datetime.fromtimestamp(0.0), "/Date(0)/"), + (datetime(1960, 1, 1, tzinfo=tz.UTC), "/Date(-315619200000+0000)/"), (datetime.fromtimestamp(1439424000.0), "/Date(1439424000000)/"), (datetime.fromtimestamp(1439434356.790), "/Date(1439434356790)/"), (datetime(2015, 8, 13, tzinfo=tz.UTC), "/Date(1439424000000+0000)/"), diff --git a/xero_python/api_client/serializer.py b/xero_python/api_client/serializer.py index cf508da9..7570a645 100644 --- a/xero_python/api_client/serializer.py +++ b/xero_python/api_client/serializer.py @@ -12,6 +12,14 @@ DICT_DATA_TYPE = re.compile(r"^dict(?:\[(.*)\])?$") LIST_DATA_TYPE = re.compile(r"^list(?:\[(.*)\])?$") TUPLE_DATA_TYPE = re.compile(r"^tuple(?:\[(.*)\])?$") +UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=tz.UTC) + + +def datetime_timestamp(value): + """Return seconds from the Unix epoch without platform timestamp limits.""" + if value.tzinfo is None: + value = value.replace(tzinfo=tz.tzlocal()) + return (value.astimezone(tz.UTC) - UNIX_EPOCH).total_seconds() def data_type(value, explicit_type=None): @@ -159,7 +167,7 @@ def serialize_datetime_ms(value, explicit_type=None): :return: serialized object """ tz_str = value.strftime("%z") - timestamp_s = value.timestamp() + timestamp_s = datetime_timestamp(value) timestamp_ms = int(timestamp_s * 1000) return "/Date({}{})/".format(timestamp_ms, tz_str) @@ -180,7 +188,7 @@ def serialize_date_ms(value, explicit_type=None): else: raise ValueError("Can't serialize {!r} into Microsoft date json format") - timestamp_s = datetime_value.timestamp() + timestamp_s = datetime_timestamp(datetime_value) timestamp_ms = int(timestamp_s * 1000) return "/Date({})/".format(timestamp_ms) From d8379405cf89db6bb2b850d8fd2c4e368887d90d Mon Sep 17 00:00:00 2001 From: Ryan Duguid <152749594+ryanduguid@users.noreply.github.com> Date: Thu, 13 Aug 2026 03:08:42 +1000 Subject: [PATCH 2/4] test: cover pre-epoch local and date values --- tests/test_api_client/test_serializer.py | 21 +++++++++++++++++++++ xero_python/api_client/serializer.py | 10 +++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/test_api_client/test_serializer.py b/tests/test_api_client/test_serializer.py index 61ce9ab4..db944237 100644 --- a/tests/test_api_client/test_serializer.py +++ b/tests/test_api_client/test_serializer.py @@ -338,6 +338,18 @@ def test_serialize_datetime_ms(value, expected): assert result == expected +def test_serialize_naive_pre_epoch_datetime_ms_uses_local_timezone(): + value = datetime(1960, 1, 1, 12, 30) + windows_local = getattr(tz, "tzwinlocal", None) + local_tz = windows_local() if windows_local is not None else tz.tzlocal() + local_offset = local_tz.utcoffset(value) or timedelta() + utc_value = (value - local_offset).replace(tzinfo=tz.UTC) + epoch = datetime(1970, 1, 1, tzinfo=tz.UTC) + expected_ms = int((utc_value - epoch).total_seconds() * 1000) + + assert serialize_datetime_ms(value) == "/Date({})/".format(expected_ms) + + # serialize_date_ms tests @pytest.mark.parametrize( "value,expected", @@ -357,6 +369,15 @@ def test_serialize_date_ms(value, expected): assert result == expected +def test_serialize_pre_epoch_date_ms_uses_utc_midnight(): + value = date(1960, 1, 1) + utc_value = datetime.combine(value, datetime.min.time()).replace(tzinfo=tz.UTC) + epoch = datetime(1970, 1, 1, tzinfo=tz.UTC) + expected_ms = int((utc_value - epoch).total_seconds() * 1000) + + assert serialize_date_ms(value) == "/Date({})/".format(expected_ms) + + # serialize_base_model tests def test_serialize_base_model(): # given test model diff --git a/xero_python/api_client/serializer.py b/xero_python/api_client/serializer.py index 7570a645..68ac22c5 100644 --- a/xero_python/api_client/serializer.py +++ b/xero_python/api_client/serializer.py @@ -15,10 +15,18 @@ UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=tz.UTC) +def local_timezone(): + """Return a local timezone that supports pre-epoch dates on Windows.""" + windows_local = getattr(tz, "tzwinlocal", None) + if windows_local is not None: + return windows_local() + return tz.tzlocal() + + def datetime_timestamp(value): """Return seconds from the Unix epoch without platform timestamp limits.""" if value.tzinfo is None: - value = value.replace(tzinfo=tz.tzlocal()) + value = value.replace(tzinfo=local_timezone()) return (value.astimezone(tz.UTC) - UNIX_EPOCH).total_seconds() From 93d24fcbbfbc423cd8b4e956259442dc50e397d1 Mon Sep 17 00:00:00 2001 From: Ryan Duguid <152749594+ryanduguid@users.noreply.github.com> Date: Thu, 13 Aug 2026 03:38:06 +1000 Subject: [PATCH 3/4] Preserve native local datetime semantics --- tests/test_api_client/test_serializer.py | 17 +++++++++++++++++ xero_python/api_client/serializer.py | 5 +++++ 2 files changed, 22 insertions(+) diff --git a/tests/test_api_client/test_serializer.py b/tests/test_api_client/test_serializer.py index db944237..fbb9b3b4 100644 --- a/tests/test_api_client/test_serializer.py +++ b/tests/test_api_client/test_serializer.py @@ -350,6 +350,23 @@ def test_serialize_naive_pre_epoch_datetime_ms_uses_local_timezone(): assert serialize_datetime_ms(value) == "/Date({})/".format(expected_ms) +@pytest.mark.parametrize( + "value", + [ + datetime(1971, 1, 1, 12, 30), + datetime(2015, 8, 13, 12, 30), + datetime(2024, 4, 7, 2, 30, fold=0), + datetime(2024, 4, 7, 2, 30, fold=1), + datetime(2024, 10, 6, 2, 30, fold=0), + datetime(2024, 10, 6, 2, 30, fold=1), + ], +) +def test_serialize_naive_datetime_ms_preserves_platform_timestamp_semantics(value): + expected_ms = int(value.timestamp() * 1000) + + assert serialize_datetime_ms(value) == "/Date({})/".format(expected_ms) + + # serialize_date_ms tests @pytest.mark.parametrize( "value,expected", diff --git a/xero_python/api_client/serializer.py b/xero_python/api_client/serializer.py index 68ac22c5..83e3012a 100644 --- a/xero_python/api_client/serializer.py +++ b/xero_python/api_client/serializer.py @@ -25,6 +25,11 @@ def local_timezone(): def datetime_timestamp(value): """Return seconds from the Unix epoch without platform timestamp limits.""" + try: + # Preserve the platform's existing naive-local DST and fold semantics. + return value.timestamp() + except (OSError, OverflowError): + pass if value.tzinfo is None: value = value.replace(tzinfo=local_timezone()) return (value.astimezone(tz.UTC) - UNIX_EPOCH).total_seconds() From c493d8182228530f0aeb074ccd381c3333316500 Mon Sep 17 00:00:00 2001 From: Ryan Duguid <152749594+ryanduguid@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:30:02 +1000 Subject: [PATCH 4/4] fix: use historical offsets and exact milliseconds for MS dates tz.tzwinlocal() reports the DST rule in force today for every date, so naive pre-epoch datetimes were converted with the wrong UTC offset and output differed by platform. Serialising 1960-01-01 12:30 in Sydney gave /Date(-315613800000)/, an hour off the true instant -315610200000, while Linux read the tz database and was correct. Resolve the machine's IANA zone name instead and take the offsets from the database bundled with python-dateutil, which adds tzlocal to the runtime requirements. Epoch conversion stayed in float seconds and truncated towards zero, losing a millisecond on values that are not exactly representable and flipping the rounding direction either side of the epoch. Use integer arithmetic. Deserialisation still went through datetime.fromtimestamp, which raises OSError on Windows for pre-epoch values, so neither Xero's own /Date(-2208988800000)/ nor this branch's new output could be read back. Offset the epoch by a timedelta instead. Replace the naive pre-epoch test, which recomputed its expectation with a copy of the implementation and so could not detect the hour shift, with cases that pin a fixed timezone to known absolute instants. Add regression cases that fail without the fix: exact millisecond values either side of the epoch, pre-epoch deserialisation, and a serialise/deserialise round trip. --- requirements.txt | 3 + tests/test_api_client/test_deserializer.py | 9 +++ tests/test_api_client/test_serializer.py | 82 +++++++++++++++++++--- xero_python/api_client/deserializer.py | 9 ++- xero_python/api_client/serializer.py | 63 ++++++++++++----- 5 files changed, 139 insertions(+), 27 deletions(-) diff --git a/requirements.txt b/requirements.txt index f1163e77..b0402081 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,8 @@ # make sure content of this file can be parsed by setup.load_requirements python-dateutil>=2.7 +# resolves the local IANA timezone name so pre-epoch datetimes use real +# historical UTC offsets on Windows, which only stores the current DST rule +tzlocal>=4.0 urllib3 certifi setuptools>=75.1.0 diff --git a/tests/test_api_client/test_deserializer.py b/tests/test_api_client/test_deserializer.py index 9c851a8a..3ed9deef 100644 --- a/tests/test_api_client/test_deserializer.py +++ b/tests/test_api_client/test_deserializer.py @@ -207,6 +207,9 @@ def test_deserialize_date_error(data): ("/Date(315619200000+0000)/", date(1980, 1, 2)), ("/Date(1550899400362)/", date(2019, 2, 23)), ("/Date(1550899400362+1300)/", date(2019, 2, 23)), + # Pre-epoch timestamps are outside the platform range Windows accepts. + ("/Date(-2208988800000)/", date(1900, 1, 1)), + ("/Date(-315619200000+0000)/", date(1960, 1, 1)), ], ) def test_deserialize_date_ms(data, expected): @@ -306,6 +309,12 @@ def test_deserialize_datetime_error(data): tzinfo=tz.tzoffset(None, timedelta(hours=13)), ), ), + # Pre-epoch timestamps are outside the platform range Windows accepts. + ("/Date(-2208988800000)/", datetime(1900, 1, 1, tzinfo=tz.UTC)), + ( + "/Date(-2150881754232+0000)/", + datetime(1901, 11, 4, 12, 50, 45, 768000, tzinfo=tz.UTC), + ), ], ) def test_deserialize_datetime_ms(data, expected): diff --git a/tests/test_api_client/test_serializer.py b/tests/test_api_client/test_serializer.py index fbb9b3b4..bfa87003 100644 --- a/tests/test_api_client/test_serializer.py +++ b/tests/test_api_client/test_serializer.py @@ -7,8 +7,11 @@ import pytest from dateutil import tz +from xero_python.api_client.deserializer import deserialize from xero_python.api_client.serializer import ( data_type, + local_timezone, + naive_to_utc, serialize, serialize_routing, serialize_dict, @@ -338,16 +341,39 @@ def test_serialize_datetime_ms(value, expected): assert result == expected -def test_serialize_naive_pre_epoch_datetime_ms_uses_local_timezone(): +@pytest.mark.parametrize( + "value,expected", + [ + # Sydney stayed on UTC+10 all year round until 1971, so noon-thirty + # local is 02:30 UTC. Applying today's daylight saving rule instead + # would place it an hour earlier, at 01:30 UTC. + (datetime(1960, 1, 1, 12, 30), datetime(1960, 1, 1, 2, 30, tzinfo=tz.UTC)), + # By 1990 daylight saving was in force each January, so UTC+11 applies. + (datetime(1990, 1, 1, 12, 30), datetime(1990, 1, 1, 1, 30, tzinfo=tz.UTC)), + ], +) +def test_naive_to_utc_uses_historical_offsets_of_a_fixed_timezone( + monkeypatch, value, expected +): + monkeypatch.setattr( + "xero_python.api_client.serializer.local_timezone", + lambda: tz.gettz("Australia/Sydney"), + ) + + assert naive_to_utc(value) == expected + + +def test_local_timezone_agrees_with_the_tz_database_before_the_epoch(): + zone_name = pytest.importorskip("tzlocal").get_localzone_name() + historical_zone = tz.gettz(zone_name) + if historical_zone is None: + pytest.skip("no tz database entry for {}".format(zone_name)) value = datetime(1960, 1, 1, 12, 30) - windows_local = getattr(tz, "tzwinlocal", None) - local_tz = windows_local() if windows_local is not None else tz.tzlocal() - local_offset = local_tz.utcoffset(value) or timedelta() - utc_value = (value - local_offset).replace(tzinfo=tz.UTC) - epoch = datetime(1970, 1, 1, tzinfo=tz.UTC) - expected_ms = int((utc_value - epoch).total_seconds() * 1000) - assert serialize_datetime_ms(value) == "/Date({})/".format(expected_ms) + assert ( + value.replace(tzinfo=local_timezone()).utcoffset() + == value.replace(tzinfo=historical_zone).utcoffset() + ) @pytest.mark.parametrize( @@ -367,6 +393,46 @@ def test_serialize_naive_datetime_ms_preserves_platform_timestamp_semantics(valu assert serialize_datetime_ms(value) == "/Date({})/".format(expected_ms) +@pytest.mark.parametrize( + "value,expected", + [ + # Float seconds truncate towards zero, so these lose the final + # millisecond and the error changes sign either side of the epoch. + ( + datetime(1901, 11, 4, 12, 50, 45, 768000, tzinfo=tz.UTC), + "/Date(-2150881754232+0000)/", + ), + ( + datetime(1935, 5, 6, 15, 50, 10, 433000, tzinfo=tz.UTC), + "/Date(-1093680589567+0000)/", + ), + ( + datetime(2004, 5, 29, 20, 46, 5, 715000, tzinfo=tz.UTC), + "/Date(1085863565715+0000)/", + ), + ], +) +def test_serialize_datetime_ms_keeps_exact_milliseconds(value, expected): + assert serialize_datetime_ms(value) == expected + + +@pytest.mark.parametrize( + "value", + [ + datetime(1900, 1, 1, tzinfo=tz.UTC), + datetime(1901, 11, 4, 12, 50, 45, 768000, tzinfo=tz.UTC), + datetime(1960, 1, 1, tzinfo=tz.UTC), + datetime(1970, 1, 1, tzinfo=tz.UTC), + datetime(2004, 5, 29, 20, 46, 5, 715000, tzinfo=tz.UTC), + datetime(2016, 10, 13, 20, 13, 36, 437000, tzinfo=tz.UTC), + ], +) +def test_datetime_ms_round_trips_through_deserialize(value): + assert ( + deserialize("datetime[ms-format]", serialize_datetime_ms(value), None) == value + ) + + # serialize_date_ms tests @pytest.mark.parametrize( "value,expected", diff --git a/xero_python/api_client/deserializer.py b/xero_python/api_client/deserializer.py index b64e9820..5a221403 100644 --- a/xero_python/api_client/deserializer.py +++ b/xero_python/api_client/deserializer.py @@ -24,6 +24,8 @@ MS_DATETIME_RE = re.compile(r"/Date\((?P-?\d+)(?P[+-]\d{2,4})?\)/$") DATE_WITH_NO_DAY_RE = re.compile(r"(\d\d\d\d)-(\d\d)") +UNIX_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=tz.UTC) + def deserialize_routing(data_type, data, model_finder): """Custom logic to find matching deserialize implementation and @@ -250,8 +252,11 @@ def deserialize_datetime_ms(data_type, data, model_finder): tz_info = tz.UTC timestamp_ms = int(match.groupdict()["timestamp"]) - timestamp_s = timestamp_ms / 1000 - return datetime.datetime.fromtimestamp(timestamp_s, tz=tz_info) + # Offsetting the epoch keeps the arithmetic exact and works for dates + # outside the platform timestamp range, which Windows rejects. + return (UNIX_EPOCH + datetime.timedelta(milliseconds=timestamp_ms)).astimezone( + tz_info + ) elif DATE_WITH_NO_DAY_RE.match(str(data)): return datetime.datetime.strptime(data + "-01", "%Y-%m-%d") else: diff --git a/xero_python/api_client/serializer.py b/xero_python/api_client/serializer.py index 83e3012a..944278f1 100644 --- a/xero_python/api_client/serializer.py +++ b/xero_python/api_client/serializer.py @@ -16,23 +16,54 @@ def local_timezone(): - """Return a local timezone that supports pre-epoch dates on Windows.""" - windows_local = getattr(tz, "tzwinlocal", None) - if windows_local is not None: - return windows_local() + """Return the local timezone including its historical UTC offsets. + + Windows only records the currently active DST rule, so tz.tzwinlocal() + applies today's rule to every historical date and shifts pre-1971 values + by an hour. Resolve the IANA name for the machine instead and read the + real transitions from the tz database bundled with python-dateutil. + """ + try: + from tzlocal import get_localzone_name + + zone_name = get_localzone_name() + except Exception: + zone_name = None + if zone_name: + zone = tz.gettz(zone_name) + if zone is not None: + return zone return tz.tzlocal() -def datetime_timestamp(value): - """Return seconds from the Unix epoch without platform timestamp limits.""" - try: - # Preserve the platform's existing naive-local DST and fold semantics. - return value.timestamp() - except (OSError, OverflowError): - pass +def naive_to_utc(value): + """Convert a naive datetime to UTC using the local historical offset.""" + return value.replace(tzinfo=local_timezone()).astimezone(tz.UTC) + + +def datetime_to_utc(value): + """Return value as an aware UTC datetime without platform range limits.""" if value.tzinfo is None: - value = value.replace(tzinfo=local_timezone()) - return (value.astimezone(tz.UTC) - UNIX_EPOCH).total_seconds() + try: + # Preserve the platform's existing naive-local DST and fold + # semantics for every date it is able to represent. + return value.astimezone(tz.UTC) + except (OSError, OverflowError, ValueError): + return naive_to_utc(value) + return value.astimezone(tz.UTC) + + +def datetime_timestamp_ms(value): + """Return whole milliseconds from the Unix epoch. + + Uses integer arithmetic throughout. Going via float seconds truncates + towards zero, which loses a millisecond on values that are not exactly + representable and flips the rounding direction either side of the epoch. + """ + elapsed = datetime_to_utc(value) - UNIX_EPOCH + return ( + elapsed.days * 86400000 + elapsed.seconds * 1000 + elapsed.microseconds // 1000 + ) def data_type(value, explicit_type=None): @@ -180,8 +211,7 @@ def serialize_datetime_ms(value, explicit_type=None): :return: serialized object """ tz_str = value.strftime("%z") - timestamp_s = datetime_timestamp(value) - timestamp_ms = int(timestamp_s * 1000) + timestamp_ms = datetime_timestamp_ms(value) return "/Date({}{})/".format(timestamp_ms, tz_str) @@ -201,8 +231,7 @@ def serialize_date_ms(value, explicit_type=None): else: raise ValueError("Can't serialize {!r} into Microsoft date json format") - timestamp_s = datetime_timestamp(datetime_value) - timestamp_ms = int(timestamp_s * 1000) + timestamp_ms = datetime_timestamp_ms(datetime_value) return "/Date({})/".format(timestamp_ms)