diff --git a/modules/archive/models.py b/modules/archive/models.py index 4296dea6..21128381 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -1,6 +1,7 @@ import json import pathlib import re +from datetime import datetime from typing import Iterable, List, Optional import pytz @@ -27,6 +28,20 @@ MATCH_URL = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE) MATCH_DATE = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE) +def parse_singlefile_saved_date(saved_date: str) -> datetime: + """Parse SingleFile's "saved date", which is the browser's local time and its offset, into UTC. + + >>> parse_singlefile_saved_date('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)') + datetime.datetime(2024, 11, 8, 1, 36, 23, tzinfo=) + """ + parts = saved_date.split(' ') + try: + return datetime.strptime(' '.join(parts[:6]), '%a %b %d %Y %H:%M:%S GMT%z').astimezone(pytz.UTC) + except ValueError: + # No offset; assume GMT. + return dates.strpdate(' '.join(parts[:5])).replace(tzinfo=pytz.timezone('GMT')) + + class Archive(Base, ModelHelper): __tablename__ = 'archive' __table_args__ = ( @@ -331,11 +346,7 @@ class Archive(Base, ModelHelper): if not self.file_group.download_datetime: try: if match := MATCH_DATE.findall(head): - dt = match[0].strip() - dt = ' '.join(dt.split(' ')[:5]) - # SingleFile uses GMT. - dt = dates.strpdate(dt).replace(tzinfo=pytz.timezone('GMT')) - self.file_group.download_datetime = dt + self.file_group.download_datetime = parse_singlefile_saved_date(match[0].strip()) except Exception as e: logger.error(f'Could not get archive date from singlefile {path}', exc_info=e) diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 0a49dee3..7c4ebe45 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -2,7 +2,7 @@ import json import pathlib import re import tempfile -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from http import HTTPStatus from unittest import mock @@ -633,6 +633,29 @@ async def test_refresh_archives_index(test_session, make_files_structure, refres assert archive.file_group.indexed is True +@pytest.mark.asyncio +@pytest.mark.parametrize('saved_date,expected', [ + ('Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)', datetime(2022, 5, 16, 23, 51, 35)), + # The browser writes its local time; the offset must be applied, not dropped. + ('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)', datetime(2024, 11, 8, 1, 36, 23)), + ('Tue Mar 03 2026 09:15:00 GMT+0200 (Eastern European Standard Time)', datetime(2026, 3, 3, 7, 15, 0)), +]) +async def test_singlefile_saved_date_timezone(test_session, make_files_structure, refresh_files, saved_date, expected): + """An Archive's download_datetime is the SingleFile "saved date" converted to UTC.""" + make_files_structure({ + 'archive/example.com/NA.html': f''' ''', + }) + + await refresh_files() + + archive: Archive = test_session.query(Archive).one() + assert archive.file_group.download_datetime == expected.replace(tzinfo=timezone.utc) + + @pytest.mark.asyncio async def test_archive_meta(async_client, test_session, make_files_structure, refresh_files): # The start of a typical singlefile html file.