diff --git a/requirements.txt b/requirements.txt index f1163e77..b0402081 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,8 @@ # make sure content of this file can be parsed by setup.load_requirements python-dateutil>=2.7 +# resolves the local IANA timezone name so pre-epoch datetimes use real +# historical UTC offsets on Windows, which only stores the current DST rule +tzlocal>=4.0 urllib3 certifi setuptools>=75.1.0 diff --git a/tests/test_api_client/test_deserializer.py b/tests/test_api_client/test_deserializer.py index 9c851a8a..3ed9deef 100644 --- a/tests/test_api_client/test_deserializer.py +++ b/tests/test_api_client/test_deserializer.py @@ -207,6 +207,9 @@ def test_deserialize_date_error(data): ("/Date(315619200000+0000)/", date(1980, 1, 2)), ("/Date(1550899400362)/", date(2019, 2, 23)), ("/Date(1550899400362+1300)/", date(2019, 2, 23)), + # Pre-epoch timestamps are outside the platform range Windows accepts. + ("/Date(-2208988800000)/", date(1900, 1, 1)), + ("/Date(-315619200000+0000)/", date(1960, 1, 1)), ], ) def test_deserialize_date_ms(data, expected): @@ -306,6 +309,12 @@ def test_deserialize_datetime_error(data): tzinfo=tz.tzoffset(None, timedelta(hours=13)), ), ), + # Pre-epoch timestamps are outside the platform range Windows accepts. + ("/Date(-2208988800000)/", datetime(1900, 1, 1, tzinfo=tz.UTC)), + ( + "/Date(-2150881754232+0000)/", + datetime(1901, 11, 4, 12, 50, 45, 768000, tzinfo=tz.UTC), + ), ], ) def test_deserialize_datetime_ms(data, expected): diff --git a/tests/test_api_client/test_serializer.py b/tests/test_api_client/test_serializer.py index b190bf15..bfa87003 100644 --- a/tests/test_api_client/test_serializer.py +++ b/tests/test_api_client/test_serializer.py @@ -7,8 +7,11 @@ import pytest from dateutil import tz +from xero_python.api_client.deserializer import deserialize from xero_python.api_client.serializer import ( data_type, + local_timezone, + naive_to_utc, serialize, serialize_routing, serialize_dict, @@ -306,6 +309,7 @@ def test_serialize_datetime(value, expected): "value,expected", [ (datetime.fromtimestamp(0.0), "/Date(0)/"), + (datetime(1960, 1, 1, tzinfo=tz.UTC), "/Date(-315619200000+0000)/"), (datetime.fromtimestamp(1439424000.0), "/Date(1439424000000)/"), (datetime.fromtimestamp(1439434356.790), "/Date(1439434356790)/"), (datetime(2015, 8, 13, tzinfo=tz.UTC), "/Date(1439424000000+0000)/"), @@ -337,6 +341,98 @@ def test_serialize_datetime_ms(value, expected): assert result == expected +@pytest.mark.parametrize( + "value,expected", + [ + # Sydney stayed on UTC+10 all year round until 1971, so noon-thirty + # local is 02:30 UTC. Applying today's daylight saving rule instead + # would place it an hour earlier, at 01:30 UTC. + (datetime(1960, 1, 1, 12, 30), datetime(1960, 1, 1, 2, 30, tzinfo=tz.UTC)), + # By 1990 daylight saving was in force each January, so UTC+11 applies. + (datetime(1990, 1, 1, 12, 30), datetime(1990, 1, 1, 1, 30, tzinfo=tz.UTC)), + ], +) +def test_naive_to_utc_uses_historical_offsets_of_a_fixed_timezone( + monkeypatch, value, expected +): + monkeypatch.setattr( + "xero_python.api_client.serializer.local_timezone", + lambda: tz.gettz("Australia/Sydney"), + ) + + assert naive_to_utc(value) == expected + + +def test_local_timezone_agrees_with_the_tz_database_before_the_epoch(): + zone_name = pytest.importorskip("tzlocal").get_localzone_name() + historical_zone = tz.gettz(zone_name) + if historical_zone is None: + pytest.skip("no tz database entry for {}".format(zone_name)) + value = datetime(1960, 1, 1, 12, 30) + + assert ( + value.replace(tzinfo=local_timezone()).utcoffset() + == value.replace(tzinfo=historical_zone).utcoffset() + ) + + +@pytest.mark.parametrize( + "value", + [ + datetime(1971, 1, 1, 12, 30), + datetime(2015, 8, 13, 12, 30), + datetime(2024, 4, 7, 2, 30, fold=0), + datetime(2024, 4, 7, 2, 30, fold=1), + datetime(2024, 10, 6, 2, 30, fold=0), + datetime(2024, 10, 6, 2, 30, fold=1), + ], +) +def test_serialize_naive_datetime_ms_preserves_platform_timestamp_semantics(value): + expected_ms = int(value.timestamp() * 1000) + + assert serialize_datetime_ms(value) == "/Date({})/".format(expected_ms) + + +@pytest.mark.parametrize( + "value,expected", + [ + # Float seconds truncate towards zero, so these lose the final + # millisecond and the error changes sign either side of the epoch. + ( + datetime(1901, 11, 4, 12, 50, 45, 768000, tzinfo=tz.UTC), + "/Date(-2150881754232+0000)/", + ), + ( + datetime(1935, 5, 6, 15, 50, 10, 433000, tzinfo=tz.UTC), + "/Date(-1093680589567+0000)/", + ), + ( + datetime(2004, 5, 29, 20, 46, 5, 715000, tzinfo=tz.UTC), + "/Date(1085863565715+0000)/", + ), + ], +) +def test_serialize_datetime_ms_keeps_exact_milliseconds(value, expected): + assert serialize_datetime_ms(value) == expected + + +@pytest.mark.parametrize( + "value", + [ + datetime(1900, 1, 1, tzinfo=tz.UTC), + datetime(1901, 11, 4, 12, 50, 45, 768000, tzinfo=tz.UTC), + datetime(1960, 1, 1, tzinfo=tz.UTC), + datetime(1970, 1, 1, tzinfo=tz.UTC), + datetime(2004, 5, 29, 20, 46, 5, 715000, tzinfo=tz.UTC), + datetime(2016, 10, 13, 20, 13, 36, 437000, tzinfo=tz.UTC), + ], +) +def test_datetime_ms_round_trips_through_deserialize(value): + assert ( + deserialize("datetime[ms-format]", serialize_datetime_ms(value), None) == value + ) + + # serialize_date_ms tests @pytest.mark.parametrize( "value,expected", @@ -356,6 +452,15 @@ def test_serialize_date_ms(value, expected): assert result == expected +def test_serialize_pre_epoch_date_ms_uses_utc_midnight(): + value = date(1960, 1, 1) + utc_value = datetime.combine(value, datetime.min.time()).replace(tzinfo=tz.UTC) + epoch = datetime(1970, 1, 1, tzinfo=tz.UTC) + expected_ms = int((utc_value - epoch).total_seconds() * 1000) + + assert serialize_date_ms(value) == "/Date({})/".format(expected_ms) + + # serialize_base_model tests def test_serialize_base_model(): # given test model diff --git a/xero_python/api_client/deserializer.py b/xero_python/api_client/deserializer.py index b64e9820..5a221403 100644 --- a/xero_python/api_client/deserializer.py +++ b/xero_python/api_client/deserializer.py @@ -24,6 +24,8 @@ MS_DATETIME_RE = re.compile(r"/Date\((?P-?\d+)(?P[+-]\d{2,4})?\)/$") DATE_WITH_NO_DAY_RE = re.compile(r"(\d\d\d\d)-(\d\d)") +UNIX_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=tz.UTC) + def deserialize_routing(data_type, data, model_finder): """Custom logic to find matching deserialize implementation and @@ -250,8 +252,11 @@ def deserialize_datetime_ms(data_type, data, model_finder): tz_info = tz.UTC timestamp_ms = int(match.groupdict()["timestamp"]) - timestamp_s = timestamp_ms / 1000 - return datetime.datetime.fromtimestamp(timestamp_s, tz=tz_info) + # Offsetting the epoch keeps the arithmetic exact and works for dates + # outside the platform timestamp range, which Windows rejects. + return (UNIX_EPOCH + datetime.timedelta(milliseconds=timestamp_ms)).astimezone( + tz_info + ) elif DATE_WITH_NO_DAY_RE.match(str(data)): return datetime.datetime.strptime(data + "-01", "%Y-%m-%d") else: diff --git a/xero_python/api_client/serializer.py b/xero_python/api_client/serializer.py index cf508da9..944278f1 100644 --- a/xero_python/api_client/serializer.py +++ b/xero_python/api_client/serializer.py @@ -12,6 +12,58 @@ DICT_DATA_TYPE = re.compile(r"^dict(?:\[(.*)\])?$") LIST_DATA_TYPE = re.compile(r"^list(?:\[(.*)\])?$") TUPLE_DATA_TYPE = re.compile(r"^tuple(?:\[(.*)\])?$") +UNIX_EPOCH = datetime(1970, 1, 1, tzinfo=tz.UTC) + + +def local_timezone(): + """Return the local timezone including its historical UTC offsets. + + Windows only records the currently active DST rule, so tz.tzwinlocal() + applies today's rule to every historical date and shifts pre-1971 values + by an hour. Resolve the IANA name for the machine instead and read the + real transitions from the tz database bundled with python-dateutil. + """ + try: + from tzlocal import get_localzone_name + + zone_name = get_localzone_name() + except Exception: + zone_name = None + if zone_name: + zone = tz.gettz(zone_name) + if zone is not None: + return zone + return tz.tzlocal() + + +def naive_to_utc(value): + """Convert a naive datetime to UTC using the local historical offset.""" + return value.replace(tzinfo=local_timezone()).astimezone(tz.UTC) + + +def datetime_to_utc(value): + """Return value as an aware UTC datetime without platform range limits.""" + if value.tzinfo is None: + try: + # Preserve the platform's existing naive-local DST and fold + # semantics for every date it is able to represent. + return value.astimezone(tz.UTC) + except (OSError, OverflowError, ValueError): + return naive_to_utc(value) + return value.astimezone(tz.UTC) + + +def datetime_timestamp_ms(value): + """Return whole milliseconds from the Unix epoch. + + Uses integer arithmetic throughout. Going via float seconds truncates + towards zero, which loses a millisecond on values that are not exactly + representable and flips the rounding direction either side of the epoch. + """ + elapsed = datetime_to_utc(value) - UNIX_EPOCH + return ( + elapsed.days * 86400000 + elapsed.seconds * 1000 + elapsed.microseconds // 1000 + ) def data_type(value, explicit_type=None): @@ -159,8 +211,7 @@ def serialize_datetime_ms(value, explicit_type=None): :return: serialized object """ tz_str = value.strftime("%z") - timestamp_s = value.timestamp() - timestamp_ms = int(timestamp_s * 1000) + timestamp_ms = datetime_timestamp_ms(value) return "/Date({}{})/".format(timestamp_ms, tz_str) @@ -180,8 +231,7 @@ def serialize_date_ms(value, explicit_type=None): else: raise ValueError("Can't serialize {!r} into Microsoft date json format") - timestamp_s = datetime_value.timestamp() - timestamp_ms = int(timestamp_s * 1000) + timestamp_ms = datetime_timestamp_ms(datetime_value) return "/Date({})/".format(timestamp_ms)