rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-25 18:40:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Read the whole SingleFile header; finish the date repair only when every Archive was read

Browse source 
The header was read in 1000 bytes, so a long URL cut the saved date's offset off, and the cut-off date was
stored as GMT.  The header is now read through the end of its comment, and a malformed offset leaves the
date unset instead of guessing.

The repair is no longer marked done when an Archive could not be read (a missing file is a deleted Archive,
not a failure), and it waits for the media to be mounted, because `refresh_complete` is restored from the
database before the drives are checked.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-10-23 14:14:05 -06:00
parent afd8f157c6
commit b43372f818
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
4 changed files with 122 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

3
main.py
View file

@ -429,6 +429,9 @@ async def perpetual_repair_archive_saved_dates():
async with flags.refresh_complete.wait_for():
# Archives must be in the database.
pass
async with flags.media_mounted.wait_for():
# Their files must be readable; `refresh_complete` is restored from the database before the drive is checked.
pass
from modules.archive.lib import repair_singlefile_saved_dates
try:

28
modules/archive/lib.py
View file

@ -19,7 +19,7 @@ import pytz
from sqlalchemy import asc, func, BigInteger
from sqlalchemy.orm import Session
from modules.archive.models import Archive, MATCH_DATE, parse_singlefile_saved_date
from modules.archive.models import Archive, MATCH_DATE, parse_singlefile_saved_date, read_singlefile_header
from wrolpi import dates
from wrolpi import fts
from wrolpi.cmd import READABILITY_BIN, run_command
@ -1861,7 +1861,8 @@ def repair_singlefile_saved_dates() -> int:
The saved date was once parsed without its offset. Only a download_datetime equal to that misread value, from a
saved date with a non-zero offset, is changed; any other download_datetime is never touched. Runs once per
SAVED_DATE_REPAIR_VERSION. Returns the number of Archives corrected."""
SAVED_DATE_REPAIR_VERSION, and is not marked done while any Archive could not be read. A missing file is a
deleted Archive, not a failure. Returns the number of Archives corrected."""
if get_kv_value(SAVED_DATE_REPAIR_KEY) == SAVED_DATE_REPAIR_VERSION:
return 0
@ -1872,13 +1873,18 @@ def repair_singlefile_saved_dates() -> int:
.all()
corrections = dict()
missing = failed = 0
for file_group_id, path, download_datetime in rows:
try:
with open(path, 'rb') as fh:
head = fh.read(1000).decode(errors='ignore')
except OSError:
head = read_singlefile_header(path)
except FileNotFoundError:
missing += 1
continue
if not (match := MATCH_DATE.search(head)):
except OSError as e:
logger.warning(f'Could not read the SingleFile header of {path}', exc_info=e)
failed += 1
continue
if not head or not (match := MATCH_DATE.search(head)):
continue
saved_date = match.group(1).strip()
try:
@ -1887,7 +1893,7 @@ def repair_singlefile_saved_dates() -> int:
misread = dates.strpdate(' '.join(saved_date.split(' ')[:5])).replace(tzinfo=pytz.timezone('GMT'))
except Exception:
continue
if corrected != misread and download_datetime == misread:
if corrected and corrected != misread and download_datetime == misread:
corrections[file_group_id] = corrected
for chunk in chunks(list(corrections.items()), 500):
@ -1895,6 +1901,10 @@ def repair_singlefile_saved_dates() -> int:
for file_group in session.query(FileGroup).filter(FileGroup.id.in_([i for i, _ in chunk])):
file_group.download_datetime = corrections[file_group.id]
set_kv_value(SAVED_DATE_REPAIR_KEY, SAVED_DATE_REPAIR_VERSION)
logger.info(f'Corrected the download datetime of {len(corrections)} Archives from their SingleFile saved date')
logger.info(f'Corrected the download datetime of {len(corrections)} Archives from their SingleFile saved date'
f' ({missing} missing)')
if failed:
logger.warning(f'Could not read {failed} Archives; their download datetimes will be repaired later')
else:
set_kv_value(SAVED_DATE_REPAIR_KEY, SAVED_DATE_REPAIR_VERSION)
return len(corrections)

61
modules/archive/models.py
View file

@ -28,18 +28,39 @@ MATCH_URL = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE)
MATCH_DATE = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE)
def parse_singlefile_saved_date(saved_date: str) -> datetime:
# A SingleFile header holds the page's URL, which may be very long.
SINGLEFILE_HEADER_MAX_BYTES = 64 * 1024
def read_singlefile_header(path: pathlib.Path) -> Optional[str]:
"""Return the start of a SingleFile, through the end of its header comment, or None if the header comment
does not end within SINGLEFILE_HEADER_MAX_BYTES. A partial header could cut off its URL or saved date."""
head = b''
with path.open('rb') as fh:
# Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail.
while len(head) < SINGLEFILE_HEADER_MAX_BYTES and (chunk := fh.read(1000)):
head += chunk
if b'-->' in head:
return head.decode(errors='ignore')
return None
def parse_singlefile_saved_date(saved_date: str) -> Optional[datetime]:
"""Parse SingleFile's "saved date", which is the browser's local time and its offset, into UTC.
Returns None if the offset is malformed; the instant is unknown.
>>> parse_singlefile_saved_date('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)')
datetime.datetime(2024, 11, 8, 1, 36, 23, tzinfo=<UTC>)
"""
parts = saved_date.split(' ')
if len(parts) < 6:
# No offset; assume GMT.
return dates.strpdate(' '.join(parts[:5])).replace(tzinfo=pytz.timezone('GMT'))
try:
return datetime.strptime(' '.join(parts[:6]), '%a %b %d %Y %H:%M:%S GMT%z').astimezone(pytz.UTC)
except ValueError:
# No offset; assume GMT.
return dates.strpdate(' '.join(parts[:5])).replace(tzinfo=pytz.timezone('GMT'))
return None
class Archive(Base, ModelHelper):
@ -329,26 +350,24 @@ class Archive(Base, ModelHelper):
# This data has already been read.
return
with path.open('rb') as fh:
# Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail.
head = fh.read(1000).decode(errors='ignore')
if 'Page saved with SingleFile' not in head:
logger.error(f'Could not find SingleFile header in {self}')
return
head = read_singlefile_header(path)
if not head or 'Page saved with SingleFile' not in head:
logger.error(f'Could not find SingleFile header in {self}')
return
if not self.file_group.url:
try:
if match := MATCH_URL.findall(head):
self.file_group.url = match[0].strip()
except Exception as e:
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
if not self.file_group.url:
try:
if match := MATCH_URL.findall(head):
self.file_group.url = match[0].strip()
except Exception as e:
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
if not self.file_group.download_datetime:
try:
if match := MATCH_DATE.findall(head):
self.file_group.download_datetime = parse_singlefile_saved_date(match[0].strip())
except Exception as e:
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
if not self.file_group.download_datetime:
try:
if match := MATCH_DATE.findall(head):
self.file_group.download_datetime = parse_singlefile_saved_date(match[0].strip())
except Exception as e:
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
def apply_domain(self):
"""Get the domain collection from the URL."""

60
modules/archive/test/test_lib.py
View file

@ -656,6 +656,66 @@ async def test_singlefile_saved_date_timezone(test_session, make_files_structure
assert archive.file_group.download_datetime == expected.replace(tzinfo=timezone.utc)
@pytest.mark.asyncio
async def test_singlefile_saved_date_after_long_url(test_session, make_files_structure, refresh_files):
"""The whole SingleFile header is read, so a long URL does not cut the saved date's offset off."""
url = 'https://example.com/' + 'a' * 900
make_files_structure({
'archive/example.com/NA.html': f'''<!DOCTYPE html> <html lang="en"><!--
Page saved with SingleFile
url: {url}
saved date: Mon May 26 2025 16:10:33 GMT-0600 (Mountain Daylight Time)
--><head><meta charset="utf-8"></head></html>''',
})
await refresh_files()
archive: Archive = test_session.query(Archive).one()
assert archive.file_group.url == url
assert archive.file_group.download_datetime == datetime(2025, 5, 26, 22, 10, 33, tzinfo=timezone.utc)
@pytest.mark.parametrize('saved_date,expected', [
('Mon May 26 2025 16:10:33 GMT-0600 (Mountain Daylight Time)', datetime(2025, 5, 26, 22, 10, 33)),
('Mon May 26 2025 16:10:33 GMT+05:30', datetime(2025, 5, 26, 10, 40, 33)),
# No offset at all is GMT.
('Mon May 26 2025 16:10:33', datetime(2025, 5, 26, 16, 10, 33)),
# A cut-off offset is unknown, not GMT.
('Mon May 26 2025 16:10:33 GMT-06', None),
])
def test_parse_singlefile_saved_date(saved_date, expected):
from modules.archive.models import parse_singlefile_saved_date
expected = expected.replace(tzinfo=timezone.utc) if expected else None
assert parse_singlefile_saved_date(saved_date) == expected
@pytest.mark.asyncio
async def test_repair_singlefile_saved_dates_unreadable(test_session, make_files_structure, refresh_files):
"""The repair is not marked done while an Archive could not be read, so it runs again."""
misread = datetime(2024, 11, 7, 18, 36, 23, tzinfo=timezone.utc)
singlefile, = make_files_structure({
'archive/example.com/NA.html': '''<!DOCTYPE html> <html lang="en"><!--
Page saved with SingleFile
url: https://example.com
saved date: Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)
--><head><meta charset="utf-8"></head></html>''',
})
await refresh_files()
test_session.query(Archive).one().file_group.download_datetime = misread
test_session.commit()
singlefile.chmod(0o000)
try:
assert lib.repair_singlefile_saved_dates() == 0
finally:
singlefile.chmod(0o644)
assert lib.repair_singlefile_saved_dates() == 1
test_session.expire_all()
assert test_session.query(Archive).one().file_group.download_datetime == \
datetime(2024, 11, 8, 1, 36, 23, tzinfo=timezone.utc)
@pytest.mark.asyncio
async def test_repair_singlefile_saved_dates(test_session, make_files_structure, refresh_files):
"""An Archive whose download_datetime is its SingleFile saved date's local time read as UTC is corrected, once.