Repair Archive download datetimes misread from a SingleFile saved date

Archives parsed before the offset fix keep the shifted download_datetime, because the parser only fills
an empty value.  A one-time background repair (gated by a wrolpi_kv version, after the first refresh)
re-reads each Archive's saved date and corrects a download_datetime that equals the offset-dropped value.
Any other download_datetime is left alone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

modules/archive/lib.py

@@ -19,20 +19,20 @@ import pytz
 from sqlalchemy import asc, func, BigInteger
 from sqlalchemy.orm import Session
 
-from modules.archive.models import Archive
+from modules.archive.models import Archive, MATCH_DATE, parse_singlefile_saved_date
 from wrolpi import dates
 from wrolpi import fts
 from wrolpi.cmd import READABILITY_BIN, run_command
 from wrolpi.collections import Collection
 from wrolpi.common import get_media_directory, get_relative_to_media_directory, logger, extract_domain, \
     escape_file_name, aiohttp_post, format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, \
-    get_wrolpi_config, html_screenshot, html_file_screenshot, ConfigFile, trim_file_name
+    get_wrolpi_config, html_screenshot, html_file_screenshot, ConfigFile, trim_file_name, chunks
 from wrolpi.dates import now, Seconds
 from wrolpi.db import get_db_session, get_db_curs, session_curs
 from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
 from wrolpi.events import Events
 from wrolpi.files.lib import handle_file_group_search_results, cached_search_total, \
-    search_filter_cache_key, count_file_groups
+    search_filter_cache_key, count_file_groups, get_kv_value, set_kv_value
 from wrolpi.files.models import FileGroup
 from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
 from wrolpi.tags import tag_append_sub_select_where
@@ -1849,3 +1849,52 @@ async def generate_archive_screenshot(archive_id: int) -> pathlib.Path:
         session.flush()
 
     return screenshot_path
+
+
+# Bump to run `repair_singlefile_saved_dates` again.
+SAVED_DATE_REPAIR_VERSION = '1'
+SAVED_DATE_REPAIR_KEY = 'archive_saved_date_repair_version'
+
+
+def repair_singlefile_saved_dates() -> int:
+    """Correct each Archive whose download_datetime is its SingleFile "saved date" local time read as UTC.
+
+    The saved date was once parsed without its offset.  Only a download_datetime equal to that misread value, from a
+    saved date with a non-zero offset, is changed; any other download_datetime is never touched.  Runs once per
+    SAVED_DATE_REPAIR_VERSION.  Returns the number of Archives corrected."""
+    if get_kv_value(SAVED_DATE_REPAIR_KEY) == SAVED_DATE_REPAIR_VERSION:
+        return 0
+
+    with get_db_session() as session:
+        rows = session.query(FileGroup.id, FileGroup.primary_path, FileGroup.download_datetime) \
+            .join(Archive, Archive.file_group_id == FileGroup.id) \
+            .filter(FileGroup.download_datetime.isnot(None)) \
+            .all()
+
+    corrections = dict()
+    for file_group_id, path, download_datetime in rows:
+        try:
+            with open(path, 'rb') as fh:
+                head = fh.read(1000).decode(errors='ignore')
+        except OSError:
+            continue
+        if not (match := MATCH_DATE.search(head)):
+            continue
+        saved_date = match.group(1).strip()
+        try:
+            corrected = parse_singlefile_saved_date(saved_date)
+            # The value the offset-dropping parser stored.
+            misread = dates.strpdate(' '.join(saved_date.split(' ')[:5])).replace(tzinfo=pytz.timezone('GMT'))
+        except Exception:
+            continue
+        if corrected != misread and download_datetime == misread:
+            corrections[file_group_id] = corrected
+
+    for chunk in chunks(list(corrections.items()), 500):
+        with get_db_session(commit=True) as session:
+            for file_group in session.query(FileGroup).filter(FileGroup.id.in_([i for i, _ in chunk])):
+                file_group.download_datetime = corrections[file_group.id]
+
+    set_kv_value(SAVED_DATE_REPAIR_KEY, SAVED_DATE_REPAIR_VERSION)
+    logger.info(f'Corrected the download datetime of {len(corrections)} Archives from their SingleFile saved date')
+    return len(corrections)

modules/archive/test/test_lib.py

@@ -656,6 +656,48 @@ async def test_singlefile_saved_date_timezone(test_session, make_files_structure
     assert archive.file_group.download_datetime == expected.replace(tzinfo=timezone.utc)
 
 
+@pytest.mark.asyncio
+async def test_repair_singlefile_saved_dates(test_session, make_files_structure, refresh_files):
+    """An Archive whose download_datetime is its SingleFile saved date's local time read as UTC is corrected, once.
+    Any other download_datetime is never changed."""
+    mountain = 'Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)'
+    make_files_structure({
+        f'archive/example.com/{name}.html': f'''<!DOCTYPE html> <html lang="en"><!--
+ Page saved with SingleFile
+ url: https://example.com/{name}
+ saved date: {saved_date}
+--><head><meta charset="utf-8"></head></html>'''
+        for name, saved_date in (('misread', mountain), ('downloaded', mountain),
+                                 ('utc', 'Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)'))
+    })
+    await refresh_files()
+
+    def file_groups() -> dict:
+        test_session.expire_all()
+        return {i.file_group.primary_path.stem: i.file_group for i in test_session.query(Archive)}
+
+    # What the offset-dropping parser stored, and a time that came from elsewhere (e.g. WROLPi's own download).
+    misread = datetime(2024, 11, 7, 18, 36, 23, tzinfo=timezone.utc)
+    downloaded = datetime(2024, 11, 8, 1, 36, 50, tzinfo=timezone.utc)
+    fgs = file_groups()
+    fgs['misread'].download_datetime = misread
+    fgs['downloaded'].download_datetime = downloaded
+    test_session.commit()
+
+    assert lib.repair_singlefile_saved_dates() == 1
+
+    fgs = file_groups()
+    assert fgs['misread'].download_datetime == datetime(2024, 11, 8, 1, 36, 23, tzinfo=timezone.utc)
+    assert fgs['downloaded'].download_datetime == downloaded
+    assert fgs['utc'].download_datetime == datetime(2022, 5, 16, 23, 51, 35, tzinfo=timezone.utc)
+
+    # The repair is done; it does not run again.
+    fgs['misread'].download_datetime = misread
+    test_session.commit()
+    assert lib.repair_singlefile_saved_dates() == 0
+    assert file_groups()['misread'].download_datetime == misread
+
+
 @pytest.mark.asyncio
 async def test_archive_meta(async_client, test_session, make_files_structure, refresh_files):
     # The start of a typical singlefile html file.