mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-25 18:40:06 -04:00
Apply the offset in a SingleFile saved date
SingleFile writes the browser's local time with its offset ("... 18:36:23 GMT-0700"). The parser dropped
the offset and treated the local time as UTC, so an uploaded Archive's download_datetime was off by the
browser's UTC offset, and a Collection reorganize renamed its files with that shifted timestamp.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
1f3762ff20
commit
3c07704b9b
2 changed files with 40 additions and 6 deletions
|
|
@ -1,6 +1,7 @@
|
|||
import json
|
||||
import pathlib
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Iterable, List, Optional
|
||||
|
||||
import pytz
|
||||
|
|
@ -27,6 +28,20 @@ MATCH_URL = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE)
|
|||
MATCH_DATE = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE)
|
||||
|
||||
|
||||
def parse_singlefile_saved_date(saved_date: str) -> datetime:
|
||||
"""Parse SingleFile's "saved date", which is the browser's local time and its offset, into UTC.
|
||||
|
||||
>>> parse_singlefile_saved_date('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)')
|
||||
datetime.datetime(2024, 11, 8, 1, 36, 23, tzinfo=<UTC>)
|
||||
"""
|
||||
parts = saved_date.split(' ')
|
||||
try:
|
||||
return datetime.strptime(' '.join(parts[:6]), '%a %b %d %Y %H:%M:%S GMT%z').astimezone(pytz.UTC)
|
||||
except ValueError:
|
||||
# No offset; assume GMT.
|
||||
return dates.strpdate(' '.join(parts[:5])).replace(tzinfo=pytz.timezone('GMT'))
|
||||
|
||||
|
||||
class Archive(Base, ModelHelper):
|
||||
__tablename__ = 'archive'
|
||||
__table_args__ = (
|
||||
|
|
@ -331,11 +346,7 @@ class Archive(Base, ModelHelper):
|
|||
if not self.file_group.download_datetime:
|
||||
try:
|
||||
if match := MATCH_DATE.findall(head):
|
||||
dt = match[0].strip()
|
||||
dt = ' '.join(dt.split(' ')[:5])
|
||||
# SingleFile uses GMT.
|
||||
dt = dates.strpdate(dt).replace(tzinfo=pytz.timezone('GMT'))
|
||||
self.file_group.download_datetime = dt
|
||||
self.file_group.download_datetime = parse_singlefile_saved_date(match[0].strip())
|
||||
except Exception as e:
|
||||
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import json
|
|||
import pathlib
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from http import HTTPStatus
|
||||
from unittest import mock
|
||||
|
||||
|
|
@ -633,6 +633,29 @@ async def test_refresh_archives_index(test_session, make_files_structure, refres
|
|||
assert archive.file_group.indexed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('saved_date,expected', [
|
||||
('Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)', datetime(2022, 5, 16, 23, 51, 35)),
|
||||
# The browser writes its local time; the offset must be applied, not dropped.
|
||||
('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)', datetime(2024, 11, 8, 1, 36, 23)),
|
||||
('Tue Mar 03 2026 09:15:00 GMT+0200 (Eastern European Standard Time)', datetime(2026, 3, 3, 7, 15, 0)),
|
||||
])
|
||||
async def test_singlefile_saved_date_timezone(test_session, make_files_structure, refresh_files, saved_date, expected):
|
||||
"""An Archive's download_datetime is the SingleFile "saved date" converted to UTC."""
|
||||
make_files_structure({
|
||||
'archive/example.com/NA.html': f'''<!DOCTYPE html> <html lang="en"><!--
|
||||
Page saved with SingleFile
|
||||
url: https://example.com
|
||||
saved date: {saved_date}
|
||||
--><head><meta charset="utf-8"></head></html>''',
|
||||
})
|
||||
|
||||
await refresh_files()
|
||||
|
||||
archive: Archive = test_session.query(Archive).one()
|
||||
assert archive.file_group.download_datetime == expected.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_meta(async_client, test_session, make_files_structure, refresh_files):
|
||||
# The start of a typical singlefile html file.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue