diff --git a/main.py b/main.py index 26731567..c6066e6a 100755 --- a/main.py +++ b/main.py @@ -429,6 +429,9 @@ async def perpetual_repair_archive_saved_dates(): async with flags.refresh_complete.wait_for(): # Archives must be in the database. pass + async with flags.media_mounted.wait_for(): + # Their files must be readable; `refresh_complete` is restored from the database before the drive is checked. + pass from modules.archive.lib import repair_singlefile_saved_dates try: diff --git a/modules/archive/lib.py b/modules/archive/lib.py index dc454a8b..6657bfdd 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -19,7 +19,7 @@ import pytz from sqlalchemy import asc, func, BigInteger from sqlalchemy.orm import Session -from modules.archive.models import Archive, MATCH_DATE, parse_singlefile_saved_date +from modules.archive.models import Archive, MATCH_DATE, parse_singlefile_saved_date, read_singlefile_header from wrolpi import dates from wrolpi import fts from wrolpi.cmd import READABILITY_BIN, run_command @@ -1861,7 +1861,8 @@ def repair_singlefile_saved_dates() -> int: The saved date was once parsed without its offset. Only a download_datetime equal to that misread value, from a saved date with a non-zero offset, is changed; any other download_datetime is never touched. Runs once per - SAVED_DATE_REPAIR_VERSION. Returns the number of Archives corrected.""" + SAVED_DATE_REPAIR_VERSION, and is not marked done while any Archive could not be read. A missing file is a + deleted Archive, not a failure. Returns the number of Archives corrected.""" if get_kv_value(SAVED_DATE_REPAIR_KEY) == SAVED_DATE_REPAIR_VERSION: return 0 @@ -1872,13 +1873,18 @@ def repair_singlefile_saved_dates() -> int: .all() corrections = dict() + missing = failed = 0 for file_group_id, path, download_datetime in rows: try: - with open(path, 'rb') as fh: - head = fh.read(1000).decode(errors='ignore') - except OSError: + head = read_singlefile_header(path) + except FileNotFoundError: + missing += 1 continue - if not (match := MATCH_DATE.search(head)): + except OSError as e: + logger.warning(f'Could not read the SingleFile header of {path}', exc_info=e) + failed += 1 + continue + if not head or not (match := MATCH_DATE.search(head)): continue saved_date = match.group(1).strip() try: @@ -1887,7 +1893,7 @@ def repair_singlefile_saved_dates() -> int: misread = dates.strpdate(' '.join(saved_date.split(' ')[:5])).replace(tzinfo=pytz.timezone('GMT')) except Exception: continue - if corrected != misread and download_datetime == misread: + if corrected and corrected != misread and download_datetime == misread: corrections[file_group_id] = corrected for chunk in chunks(list(corrections.items()), 500): @@ -1895,6 +1901,10 @@ def repair_singlefile_saved_dates() -> int: for file_group in session.query(FileGroup).filter(FileGroup.id.in_([i for i, _ in chunk])): file_group.download_datetime = corrections[file_group.id] - set_kv_value(SAVED_DATE_REPAIR_KEY, SAVED_DATE_REPAIR_VERSION) - logger.info(f'Corrected the download datetime of {len(corrections)} Archives from their SingleFile saved date') + logger.info(f'Corrected the download datetime of {len(corrections)} Archives from their SingleFile saved date' + f' ({missing} missing)') + if failed: + logger.warning(f'Could not read {failed} Archives; their download datetimes will be repaired later') + else: + set_kv_value(SAVED_DATE_REPAIR_KEY, SAVED_DATE_REPAIR_VERSION) return len(corrections) diff --git a/modules/archive/models.py b/modules/archive/models.py index 21128381..6f95ecf1 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -28,18 +28,39 @@ MATCH_URL = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE) MATCH_DATE = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE) -def parse_singlefile_saved_date(saved_date: str) -> datetime: +# A SingleFile header holds the page's URL, which may be very long. +SINGLEFILE_HEADER_MAX_BYTES = 64 * 1024 + + +def read_singlefile_header(path: pathlib.Path) -> Optional[str]: + """Return the start of a SingleFile, through the end of its header comment, or None if the header comment + does not end within SINGLEFILE_HEADER_MAX_BYTES. A partial header could cut off its URL or saved date.""" + head = b'' + with path.open('rb') as fh: + # Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail. + while len(head) < SINGLEFILE_HEADER_MAX_BYTES and (chunk := fh.read(1000)): + head += chunk + if b'-->' in head: + return head.decode(errors='ignore') + return None + + +def parse_singlefile_saved_date(saved_date: str) -> Optional[datetime]: """Parse SingleFile's "saved date", which is the browser's local time and its offset, into UTC. + Returns None if the offset is malformed; the instant is unknown. + >>> parse_singlefile_saved_date('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)') datetime.datetime(2024, 11, 8, 1, 36, 23, tzinfo=) """ parts = saved_date.split(' ') + if len(parts) < 6: + # No offset; assume GMT. + return dates.strpdate(' '.join(parts[:5])).replace(tzinfo=pytz.timezone('GMT')) try: return datetime.strptime(' '.join(parts[:6]), '%a %b %d %Y %H:%M:%S GMT%z').astimezone(pytz.UTC) except ValueError: - # No offset; assume GMT. - return dates.strpdate(' '.join(parts[:5])).replace(tzinfo=pytz.timezone('GMT')) + return None class Archive(Base, ModelHelper): @@ -329,26 +350,24 @@ class Archive(Base, ModelHelper): # This data has already been read. return - with path.open('rb') as fh: - # Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail. - head = fh.read(1000).decode(errors='ignore') - if 'Page saved with SingleFile' not in head: - logger.error(f'Could not find SingleFile header in {self}') - return + head = read_singlefile_header(path) + if not head or 'Page saved with SingleFile' not in head: + logger.error(f'Could not find SingleFile header in {self}') + return - if not self.file_group.url: - try: - if match := MATCH_URL.findall(head): - self.file_group.url = match[0].strip() - except Exception as e: - logger.error(f'Could not get URL from singlefile {path}', exc_info=e) + if not self.file_group.url: + try: + if match := MATCH_URL.findall(head): + self.file_group.url = match[0].strip() + except Exception as e: + logger.error(f'Could not get URL from singlefile {path}', exc_info=e) - if not self.file_group.download_datetime: - try: - if match := MATCH_DATE.findall(head): - self.file_group.download_datetime = parse_singlefile_saved_date(match[0].strip()) - except Exception as e: - logger.error(f'Could not get archive date from singlefile {path}', exc_info=e) + if not self.file_group.download_datetime: + try: + if match := MATCH_DATE.findall(head): + self.file_group.download_datetime = parse_singlefile_saved_date(match[0].strip()) + except Exception as e: + logger.error(f'Could not get archive date from singlefile {path}', exc_info=e) def apply_domain(self): """Get the domain collection from the URL.""" diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index f5923c3a..316f58b1 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -656,6 +656,66 @@ async def test_singlefile_saved_date_timezone(test_session, make_files_structure assert archive.file_group.download_datetime == expected.replace(tzinfo=timezone.utc) +@pytest.mark.asyncio +async def test_singlefile_saved_date_after_long_url(test_session, make_files_structure, refresh_files): + """The whole SingleFile header is read, so a long URL does not cut the saved date's offset off.""" + url = 'https://example.com/' + 'a' * 900 + make_files_structure({ + 'archive/example.com/NA.html': f''' ''', + }) + + await refresh_files() + + archive: Archive = test_session.query(Archive).one() + assert archive.file_group.url == url + assert archive.file_group.download_datetime == datetime(2025, 5, 26, 22, 10, 33, tzinfo=timezone.utc) + + +@pytest.mark.parametrize('saved_date,expected', [ + ('Mon May 26 2025 16:10:33 GMT-0600 (Mountain Daylight Time)', datetime(2025, 5, 26, 22, 10, 33)), + ('Mon May 26 2025 16:10:33 GMT+05:30', datetime(2025, 5, 26, 10, 40, 33)), + # No offset at all is GMT. + ('Mon May 26 2025 16:10:33', datetime(2025, 5, 26, 16, 10, 33)), + # A cut-off offset is unknown, not GMT. + ('Mon May 26 2025 16:10:33 GMT-06', None), +]) +def test_parse_singlefile_saved_date(saved_date, expected): + from modules.archive.models import parse_singlefile_saved_date + expected = expected.replace(tzinfo=timezone.utc) if expected else None + assert parse_singlefile_saved_date(saved_date) == expected + + +@pytest.mark.asyncio +async def test_repair_singlefile_saved_dates_unreadable(test_session, make_files_structure, refresh_files): + """The repair is not marked done while an Archive could not be read, so it runs again.""" + misread = datetime(2024, 11, 7, 18, 36, 23, tzinfo=timezone.utc) + singlefile, = make_files_structure({ + 'archive/example.com/NA.html': ''' ''', + }) + await refresh_files() + test_session.query(Archive).one().file_group.download_datetime = misread + test_session.commit() + + singlefile.chmod(0o000) + try: + assert lib.repair_singlefile_saved_dates() == 0 + finally: + singlefile.chmod(0o644) + + assert lib.repair_singlefile_saved_dates() == 1 + test_session.expire_all() + assert test_session.query(Archive).one().file_group.download_datetime == \ + datetime(2024, 11, 8, 1, 36, 23, tzinfo=timezone.utc) + + @pytest.mark.asyncio async def test_repair_singlefile_saved_dates(test_session, make_files_structure, refresh_files): """An Archive whose download_datetime is its SingleFile saved date's local time read as UTC is corrected, once.