Apply the offset in a SingleFile saved date, and repair misread Archive dates (#862)

* Apply the offset in a SingleFile saved date

SingleFile writes the browser's local time with its offset ("... 18:36:23 GMT-0700").  The parser dropped
the offset and treated the local time as UTC, so an uploaded Archive's download_datetime was off by the
browser's UTC offset, and a Collection reorganize renamed its files with that shifted timestamp.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Repair Archive download datetimes misread from a SingleFile saved date

Archives parsed before the offset fix keep the shifted download_datetime, because the parser only fills
an empty value.  A one-time background repair (gated by a wrolpi_kv version, after the first refresh)
re-reads each Archive's saved date and corrects a download_datetime that equals the offset-dropped value.
Any other download_datetime is left alone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Read the whole SingleFile header; finish the date repair only when every Archive was read

The header was read in 1000 bytes, so a long URL cut the saved date's offset off, and the cut-off date was
stored as GMT.  The header is now read through the end of its comment, and a malformed offset leaves the
date unset instead of guessing.

The repair is no longer marked done when an Archive could not be read (a missing file is a deleted Archive,
not a failure), and it waits for the media to be mounted, because `refresh_complete` is restored from the
database before the drives are checked.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

main.py

@@ -423,6 +423,23 @@ async def perpetual_start_video_missing_comments_download():
         logger.debug('Waiting for downloads to be enabled before downloading comments...')
 
 
+@perpetual_signal(sleep=3600)
+async def perpetual_repair_archive_saved_dates():
+    """Correct Archive download datetimes that were misread from their SingleFile saved date.  Runs once."""
+    async with flags.refresh_complete.wait_for():
+        # Archives must be in the database.
+        pass
+    async with flags.media_mounted.wait_for():
+        # Their files must be readable; `refresh_complete` is restored from the database before the drive is checked.
+        pass
+
+    from modules.archive.lib import repair_singlefile_saved_dates
+    try:
+        await asyncio.to_thread(repair_singlefile_saved_dates)
+    except Exception as e:
+        logger.error('Failed to repair Archive download datetimes', exc_info=e)
+
+
 @perpetual_signal(sleep=3600)  # Check hourly
 async def perpetual_check_for_updates():
     """

modules/archive/lib.py

@@ -19,20 +19,20 @@ import pytz
 from sqlalchemy import asc, func, BigInteger
 from sqlalchemy.orm import Session
 
-from modules.archive.models import Archive
+from modules.archive.models import Archive, MATCH_DATE, parse_singlefile_saved_date, read_singlefile_header
 from wrolpi import dates
 from wrolpi import fts
 from wrolpi.cmd import READABILITY_BIN, run_command
 from wrolpi.collections import Collection
 from wrolpi.common import get_media_directory, get_relative_to_media_directory, logger, extract_domain, \
     escape_file_name, aiohttp_post, format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, \
-    get_wrolpi_config, html_screenshot, html_file_screenshot, ConfigFile, trim_file_name
+    get_wrolpi_config, html_screenshot, html_file_screenshot, ConfigFile, trim_file_name, chunks
 from wrolpi.dates import now, Seconds
 from wrolpi.db import get_db_session, get_db_curs, session_curs
 from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
 from wrolpi.events import Events
 from wrolpi.files.lib import handle_file_group_search_results, cached_search_total, \
-    search_filter_cache_key, count_file_groups
+    search_filter_cache_key, count_file_groups, get_kv_value, set_kv_value
 from wrolpi.files.models import FileGroup
 from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
 from wrolpi.tags import tag_append_sub_select_where
@@ -1849,3 +1849,62 @@ async def generate_archive_screenshot(archive_id: int) -> pathlib.Path:
         session.flush()
 
     return screenshot_path
+
+
+# Bump to run `repair_singlefile_saved_dates` again.
+SAVED_DATE_REPAIR_VERSION = '1'
+SAVED_DATE_REPAIR_KEY = 'archive_saved_date_repair_version'
+
+
+def repair_singlefile_saved_dates() -> int:
+    """Correct each Archive whose download_datetime is its SingleFile "saved date" local time read as UTC.
+
+    The saved date was once parsed without its offset.  Only a download_datetime equal to that misread value, from a
+    saved date with a non-zero offset, is changed; any other download_datetime is never touched.  Runs once per
+    SAVED_DATE_REPAIR_VERSION, and is not marked done while any Archive could not be read.  A missing file is a
+    deleted Archive, not a failure.  Returns the number of Archives corrected."""
+    if get_kv_value(SAVED_DATE_REPAIR_KEY) == SAVED_DATE_REPAIR_VERSION:
+        return 0
+
+    with get_db_session() as session:
+        rows = session.query(FileGroup.id, FileGroup.primary_path, FileGroup.download_datetime) \
+            .join(Archive, Archive.file_group_id == FileGroup.id) \
+            .filter(FileGroup.download_datetime.isnot(None)) \
+            .all()
+
+    corrections = dict()
+    missing = failed = 0
+    for file_group_id, path, download_datetime in rows:
+        try:
+            head = read_singlefile_header(path)
+        except FileNotFoundError:
+            missing += 1
+            continue
+        except OSError as e:
+            logger.warning(f'Could not read the SingleFile header of {path}', exc_info=e)
+            failed += 1
+            continue
+        if not head or not (match := MATCH_DATE.search(head)):
+            continue
+        saved_date = match.group(1).strip()
+        try:
+            corrected = parse_singlefile_saved_date(saved_date)
+            # The value the offset-dropping parser stored.
+            misread = dates.strpdate(' '.join(saved_date.split(' ')[:5])).replace(tzinfo=pytz.timezone('GMT'))
+        except Exception:
+            continue
+        if corrected and corrected != misread and download_datetime == misread:
+            corrections[file_group_id] = corrected
+
+    for chunk in chunks(list(corrections.items()), 500):
+        with get_db_session(commit=True) as session:
+            for file_group in session.query(FileGroup).filter(FileGroup.id.in_([i for i, _ in chunk])):
+                file_group.download_datetime = corrections[file_group.id]
+
+    logger.info(f'Corrected the download datetime of {len(corrections)} Archives from their SingleFile saved date'
+                f' ({missing} missing)')
+    if failed:
+        logger.warning(f'Could not read {failed} Archives; their download datetimes will be repaired later')
+    else:
+        set_kv_value(SAVED_DATE_REPAIR_KEY, SAVED_DATE_REPAIR_VERSION)
+    return len(corrections)

modules/archive/models.py

@@ -1,6 +1,7 @@
 import json
 import pathlib
 import re
+from datetime import datetime
 from typing import Iterable, List, Optional
 
 import pytz
@@ -27,6 +28,41 @@ MATCH_URL = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE)
 MATCH_DATE = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE)
 
 
+# A SingleFile header holds the page's URL, which may be very long.
+SINGLEFILE_HEADER_MAX_BYTES = 64 * 1024
+
+
+def read_singlefile_header(path: pathlib.Path) -> Optional[str]:
+    """Return the start of a SingleFile, through the end of its header comment, or None if the header comment
+    does not end within SINGLEFILE_HEADER_MAX_BYTES.  A partial header could cut off its URL or saved date."""
+    head = b''
+    with path.open('rb') as fh:
+        # Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail.
+        while len(head) < SINGLEFILE_HEADER_MAX_BYTES and (chunk := fh.read(1000)):
+            head += chunk
+            if b'-->' in head:
+                return head.decode(errors='ignore')
+    return None
+
+
+def parse_singlefile_saved_date(saved_date: str) -> Optional[datetime]:
+    """Parse SingleFile's "saved date", which is the browser's local time and its offset, into UTC.
+
+    Returns None if the offset is malformed; the instant is unknown.
+
+    >>> parse_singlefile_saved_date('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)')
+    datetime.datetime(2024, 11, 8, 1, 36, 23, tzinfo=<UTC>)
+    """
+    parts = saved_date.split(' ')
+    if len(parts) < 6:
+        # No offset; assume GMT.
+        return dates.strpdate(' '.join(parts[:5])).replace(tzinfo=pytz.timezone('GMT'))
+    try:
+        return datetime.strptime(' '.join(parts[:6]), '%a %b %d %Y %H:%M:%S GMT%z').astimezone(pytz.UTC)
+    except ValueError:
+        return None
+
+
 class Archive(Base, ModelHelper):
     __tablename__ = 'archive'
     __table_args__ = (
@@ -314,30 +350,24 @@ class Archive(Base, ModelHelper):
             # This data has already been read.
             return
 
-        with path.open('rb') as fh:
+        head = read_singlefile_header(path)
-            # Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail.
+        if not head or 'Page saved with SingleFile' not in head:
-            head = fh.read(1000).decode(errors='ignore')
+            logger.error(f'Could not find SingleFile header in {self}')
-            if 'Page saved with SingleFile' not in head:
+            return
-                logger.error(f'Could not find SingleFile header in {self}')
-                return
 
-            if not self.file_group.url:
+        if not self.file_group.url:
-                try:
+            try:
-                    if match := MATCH_URL.findall(head):
+                if match := MATCH_URL.findall(head):
-                        self.file_group.url = match[0].strip()
+                    self.file_group.url = match[0].strip()
-                except Exception as e:
+            except Exception as e:
-                    logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
+                logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
 
-            if not self.file_group.download_datetime:
+        if not self.file_group.download_datetime:
-                try:
+            try:
-                    if match := MATCH_DATE.findall(head):
+                if match := MATCH_DATE.findall(head):
-                        dt = match[0].strip()
+                    self.file_group.download_datetime = parse_singlefile_saved_date(match[0].strip())
-                        dt = ' '.join(dt.split(' ')[:5])
+            except Exception as e:
-                        # SingleFile uses GMT.
+                logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
-                        dt = dates.strpdate(dt).replace(tzinfo=pytz.timezone('GMT'))
-                        self.file_group.download_datetime = dt
-                except Exception as e:
-                    logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
 
     def apply_domain(self):
         """Get the domain collection from the URL."""

modules/archive/test/test_lib.py

@@ -2,7 +2,7 @@ import json
 import pathlib
 import re
 import tempfile
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from http import HTTPStatus
 from unittest import mock
 
@@ -633,6 +633,131 @@ async def test_refresh_archives_index(test_session, make_files_structure, refres
     assert archive.file_group.indexed is True
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize('saved_date,expected', [
+    ('Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)', datetime(2022, 5, 16, 23, 51, 35)),
+    # The browser writes its local time; the offset must be applied, not dropped.
+    ('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)', datetime(2024, 11, 8, 1, 36, 23)),
+    ('Tue Mar 03 2026 09:15:00 GMT+0200 (Eastern European Standard Time)', datetime(2026, 3, 3, 7, 15, 0)),
+])
+async def test_singlefile_saved_date_timezone(test_session, make_files_structure, refresh_files, saved_date, expected):
+    """An Archive's download_datetime is the SingleFile "saved date" converted to UTC."""
+    make_files_structure({
+        'archive/example.com/NA.html': f'''<!DOCTYPE html> <html lang="en"><!--
+ Page saved with SingleFile
+ url: https://example.com
+ saved date: {saved_date}
+--><head><meta charset="utf-8"></head></html>''',
+    })
+
+    await refresh_files()
+
+    archive: Archive = test_session.query(Archive).one()
+    assert archive.file_group.download_datetime == expected.replace(tzinfo=timezone.utc)
+
+
+@pytest.mark.asyncio
+async def test_singlefile_saved_date_after_long_url(test_session, make_files_structure, refresh_files):
+    """The whole SingleFile header is read, so a long URL does not cut the saved date's offset off."""
+    url = 'https://example.com/' + 'a' * 900
+    make_files_structure({
+        'archive/example.com/NA.html': f'''<!DOCTYPE html> <html lang="en"><!--
+ Page saved with SingleFile
+ url: {url}
+ saved date: Mon May 26 2025 16:10:33 GMT-0600 (Mountain Daylight Time)
+--><head><meta charset="utf-8"></head></html>''',
+    })
+
+    await refresh_files()
+
+    archive: Archive = test_session.query(Archive).one()
+    assert archive.file_group.url == url
+    assert archive.file_group.download_datetime == datetime(2025, 5, 26, 22, 10, 33, tzinfo=timezone.utc)
+
+
+@pytest.mark.parametrize('saved_date,expected', [
+    ('Mon May 26 2025 16:10:33 GMT-0600 (Mountain Daylight Time)', datetime(2025, 5, 26, 22, 10, 33)),
+    ('Mon May 26 2025 16:10:33 GMT+05:30', datetime(2025, 5, 26, 10, 40, 33)),
+    # No offset at all is GMT.
+    ('Mon May 26 2025 16:10:33', datetime(2025, 5, 26, 16, 10, 33)),
+    # A cut-off offset is unknown, not GMT.
+    ('Mon May 26 2025 16:10:33 GMT-06', None),
+])
+def test_parse_singlefile_saved_date(saved_date, expected):
+    from modules.archive.models import parse_singlefile_saved_date
+    expected = expected.replace(tzinfo=timezone.utc) if expected else None
+    assert parse_singlefile_saved_date(saved_date) == expected
+
+
+@pytest.mark.asyncio
+async def test_repair_singlefile_saved_dates_unreadable(test_session, make_files_structure, refresh_files):
+    """The repair is not marked done while an Archive could not be read, so it runs again."""
+    misread = datetime(2024, 11, 7, 18, 36, 23, tzinfo=timezone.utc)
+    singlefile, = make_files_structure({
+        'archive/example.com/NA.html': '''<!DOCTYPE html> <html lang="en"><!--
+ Page saved with SingleFile
+ url: https://example.com
+ saved date: Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)
+--><head><meta charset="utf-8"></head></html>''',
+    })
+    await refresh_files()
+    test_session.query(Archive).one().file_group.download_datetime = misread
+    test_session.commit()
+
+    singlefile.chmod(0o000)
+    try:
+        assert lib.repair_singlefile_saved_dates() == 0
+    finally:
+        singlefile.chmod(0o644)
+
+    assert lib.repair_singlefile_saved_dates() == 1
+    test_session.expire_all()
+    assert test_session.query(Archive).one().file_group.download_datetime == \
+           datetime(2024, 11, 8, 1, 36, 23, tzinfo=timezone.utc)
+
+
+@pytest.mark.asyncio
+async def test_repair_singlefile_saved_dates(test_session, make_files_structure, refresh_files):
+    """An Archive whose download_datetime is its SingleFile saved date's local time read as UTC is corrected, once.
+    Any other download_datetime is never changed."""
+    mountain = 'Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)'
+    make_files_structure({
+        f'archive/example.com/{name}.html': f'''<!DOCTYPE html> <html lang="en"><!--
+ Page saved with SingleFile
+ url: https://example.com/{name}
+ saved date: {saved_date}
+--><head><meta charset="utf-8"></head></html>'''
+        for name, saved_date in (('misread', mountain), ('downloaded', mountain),
+                                 ('utc', 'Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)'))
+    })
+    await refresh_files()
+
+    def file_groups() -> dict:
+        test_session.expire_all()
+        return {i.file_group.primary_path.stem: i.file_group for i in test_session.query(Archive)}
+
+    # What the offset-dropping parser stored, and a time that came from elsewhere (e.g. WROLPi's own download).
+    misread = datetime(2024, 11, 7, 18, 36, 23, tzinfo=timezone.utc)
+    downloaded = datetime(2024, 11, 8, 1, 36, 50, tzinfo=timezone.utc)
+    fgs = file_groups()
+    fgs['misread'].download_datetime = misread
+    fgs['downloaded'].download_datetime = downloaded
+    test_session.commit()
+
+    assert lib.repair_singlefile_saved_dates() == 1
+
+    fgs = file_groups()
+    assert fgs['misread'].download_datetime == datetime(2024, 11, 8, 1, 36, 23, tzinfo=timezone.utc)
+    assert fgs['downloaded'].download_datetime == downloaded
+    assert fgs['utc'].download_datetime == datetime(2022, 5, 16, 23, 51, 35, tzinfo=timezone.utc)
+
+    # The repair is done; it does not run again.
+    fgs['misread'].download_datetime = misread
+    test_session.commit()
+    assert lib.repair_singlefile_saved_dates() == 0
+    assert file_groups()['misread'].download_datetime == misread
+
+
 @pytest.mark.asyncio
 async def test_archive_meta(async_client, test_session, make_files_structure, refresh_files):
     # The start of a typical singlefile html file.

wrolpi/files/lib.py

@@ -17,7 +17,7 @@ import urllib.parse
 import zipfile
 from itertools import zip_longest
 from pathlib import Path
-from typing import Callable, List, Tuple, Union, Dict, Generator, Iterable, Set
+from typing import Callable, List, Optional, Tuple, Union, Dict, Generator, Iterable, Set
 
 import cachetools.func
 from sqlalchemy import asc, or_, text as sa_text
@@ -59,7 +59,7 @@ __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix',
            'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories',
            'get_normalized_ignored_directories', 'sanitize_ignored_directories',
            'cached_search_total', 'search_filter_cache_key', 'count_file_groups',
-           'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems',
+           'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems', 'get_kv_value', 'set_kv_value',
            'backfill_null_file_group_stems', 'dedupe_file_groups_by_directory_stem', 'merge_file_group_relations',
            'merge_into_surviving_duplicates']
 
@@ -913,25 +913,35 @@ def _ensure_kv_table(curs):
     curs.execute('CREATE TABLE IF NOT EXISTS wrolpi_kv (key TEXT PRIMARY KEY, value TEXT NOT NULL)')
 
 
-def get_stem_algorithm_version() -> int:
+def get_kv_value(key: str) -> Optional[str]:
-    """The grouping-algorithm version last applied to file_group.stem, or 0 if never stored."""
+    """The value stored in wrolpi_kv for `key`, or None."""
     with get_db_curs() as curs:
         _ensure_kv_table(curs)
-        curs.execute('SELECT value FROM wrolpi_kv WHERE key = ?', (STEM_ALGORITHM_VERSION_KEY,))
+        curs.execute('SELECT value FROM wrolpi_kv WHERE key = ?', (key,))
         row = curs.fetchone()
-        return int(row['value']) if row else 0
+        return row['value'] if row else None
 
 
-def set_stem_algorithm_version(version: int):
+def set_kv_value(key: str, value: str):
     with get_db_curs(commit=True) as curs:
         _ensure_kv_table(curs)
         curs.execute(
             'INSERT INTO wrolpi_kv (key, value) VALUES (?, ?) '
             'ON CONFLICT(key) DO UPDATE SET value = excluded.value',
-            (STEM_ALGORITHM_VERSION_KEY, str(version)),
+            (key, value),
         )
 
 
+def get_stem_algorithm_version() -> int:
+    """The grouping-algorithm version last applied to file_group.stem, or 0 if never stored."""
+    value = get_kv_value(STEM_ALGORITHM_VERSION_KEY)
+    return int(value) if value else 0
+
+
+def set_stem_algorithm_version(version: int):
+    set_kv_value(STEM_ALGORITHM_VERSION_KEY, str(version))
+
+
 def ensure_file_group_stems():
     """Recompute file_group.stem when STEM_ALGORITHM_VERSION has changed.