From da3dadde074adc2ecfe6bacf2f7f84424f903e2e Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:34:29 -0600 Subject: [PATCH] Apply the offset in a SingleFile saved date, and repair misread Archive dates (#862) * Apply the offset in a SingleFile saved date SingleFile writes the browser's local time with its offset ("... 18:36:23 GMT-0700"). The parser dropped the offset and treated the local time as UTC, so an uploaded Archive's download_datetime was off by the browser's UTC offset, and a Collection reorganize renamed its files with that shifted timestamp. Co-Authored-By: Claude Opus 5.5 * Repair Archive download datetimes misread from a SingleFile saved date Archives parsed before the offset fix keep the shifted download_datetime, because the parser only fills an empty value. A one-time background repair (gated by a wrolpi_kv version, after the first refresh) re-reads each Archive's saved date and corrects a download_datetime that equals the offset-dropped value. Any other download_datetime is left alone. Co-Authored-By: Claude Opus 5.5 * Read the whole SingleFile header; finish the date repair only when every Archive was read The header was read in 1000 bytes, so a long URL cut the saved date's offset off, and the cut-off date was stored as GMT. The header is now read through the end of its comment, and a malformed offset leaves the date unset instead of guessing. The repair is no longer marked done when an Archive could not be read (a missing file is a deleted Archive, not a failure), and it waits for the media to be mounted, because `refresh_complete` is restored from the database before the drives are checked. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- main.py | 17 +++++ modules/archive/lib.py | 65 +++++++++++++++- modules/archive/models.py | 74 ++++++++++++------ modules/archive/test/test_lib.py | 127 ++++++++++++++++++++++++++++++- wrolpi/files/lib.py | 26 +++++-- 5 files changed, 275 insertions(+), 34 deletions(-) diff --git a/main.py b/main.py index e29d49ff..c6066e6a 100755 --- a/main.py +++ b/main.py @@ -423,6 +423,23 @@ async def perpetual_start_video_missing_comments_download(): logger.debug('Waiting for downloads to be enabled before downloading comments...') +@perpetual_signal(sleep=3600) +async def perpetual_repair_archive_saved_dates(): + """Correct Archive download datetimes that were misread from their SingleFile saved date. Runs once.""" + async with flags.refresh_complete.wait_for(): + # Archives must be in the database. + pass + async with flags.media_mounted.wait_for(): + # Their files must be readable; `refresh_complete` is restored from the database before the drive is checked. + pass + + from modules.archive.lib import repair_singlefile_saved_dates + try: + await asyncio.to_thread(repair_singlefile_saved_dates) + except Exception as e: + logger.error('Failed to repair Archive download datetimes', exc_info=e) + + @perpetual_signal(sleep=3600) # Check hourly async def perpetual_check_for_updates(): """ diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 4833091b..6657bfdd 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -19,20 +19,20 @@ import pytz from sqlalchemy import asc, func, BigInteger from sqlalchemy.orm import Session -from modules.archive.models import Archive +from modules.archive.models import Archive, MATCH_DATE, parse_singlefile_saved_date, read_singlefile_header from wrolpi import dates from wrolpi import fts from wrolpi.cmd import READABILITY_BIN, run_command from wrolpi.collections import Collection from wrolpi.common import get_media_directory, get_relative_to_media_directory, logger, extract_domain, \ escape_file_name, aiohttp_post, format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, \ - get_wrolpi_config, html_screenshot, html_file_screenshot, ConfigFile, trim_file_name + get_wrolpi_config, html_screenshot, html_file_screenshot, ConfigFile, trim_file_name, chunks from wrolpi.dates import now, Seconds from wrolpi.db import get_db_session, get_db_curs, session_curs from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime from wrolpi.events import Events from wrolpi.files.lib import handle_file_group_search_results, cached_search_total, \ - search_filter_cache_key, count_file_groups + search_filter_cache_key, count_file_groups, get_kv_value, set_kv_value from wrolpi.files.models import FileGroup from wrolpi.switches import register_switch_handler, ActivateSwitchMethod from wrolpi.tags import tag_append_sub_select_where @@ -1849,3 +1849,62 @@ async def generate_archive_screenshot(archive_id: int) -> pathlib.Path: session.flush() return screenshot_path + + +# Bump to run `repair_singlefile_saved_dates` again. +SAVED_DATE_REPAIR_VERSION = '1' +SAVED_DATE_REPAIR_KEY = 'archive_saved_date_repair_version' + + +def repair_singlefile_saved_dates() -> int: + """Correct each Archive whose download_datetime is its SingleFile "saved date" local time read as UTC. + + The saved date was once parsed without its offset. Only a download_datetime equal to that misread value, from a + saved date with a non-zero offset, is changed; any other download_datetime is never touched. Runs once per + SAVED_DATE_REPAIR_VERSION, and is not marked done while any Archive could not be read. A missing file is a + deleted Archive, not a failure. Returns the number of Archives corrected.""" + if get_kv_value(SAVED_DATE_REPAIR_KEY) == SAVED_DATE_REPAIR_VERSION: + return 0 + + with get_db_session() as session: + rows = session.query(FileGroup.id, FileGroup.primary_path, FileGroup.download_datetime) \ + .join(Archive, Archive.file_group_id == FileGroup.id) \ + .filter(FileGroup.download_datetime.isnot(None)) \ + .all() + + corrections = dict() + missing = failed = 0 + for file_group_id, path, download_datetime in rows: + try: + head = read_singlefile_header(path) + except FileNotFoundError: + missing += 1 + continue + except OSError as e: + logger.warning(f'Could not read the SingleFile header of {path}', exc_info=e) + failed += 1 + continue + if not head or not (match := MATCH_DATE.search(head)): + continue + saved_date = match.group(1).strip() + try: + corrected = parse_singlefile_saved_date(saved_date) + # The value the offset-dropping parser stored. + misread = dates.strpdate(' '.join(saved_date.split(' ')[:5])).replace(tzinfo=pytz.timezone('GMT')) + except Exception: + continue + if corrected and corrected != misread and download_datetime == misread: + corrections[file_group_id] = corrected + + for chunk in chunks(list(corrections.items()), 500): + with get_db_session(commit=True) as session: + for file_group in session.query(FileGroup).filter(FileGroup.id.in_([i for i, _ in chunk])): + file_group.download_datetime = corrections[file_group.id] + + logger.info(f'Corrected the download datetime of {len(corrections)} Archives from their SingleFile saved date' + f' ({missing} missing)') + if failed: + logger.warning(f'Could not read {failed} Archives; their download datetimes will be repaired later') + else: + set_kv_value(SAVED_DATE_REPAIR_KEY, SAVED_DATE_REPAIR_VERSION) + return len(corrections) diff --git a/modules/archive/models.py b/modules/archive/models.py index 4296dea6..6f95ecf1 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -1,6 +1,7 @@ import json import pathlib import re +from datetime import datetime from typing import Iterable, List, Optional import pytz @@ -27,6 +28,41 @@ MATCH_URL = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE) MATCH_DATE = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE) +# A SingleFile header holds the page's URL, which may be very long. +SINGLEFILE_HEADER_MAX_BYTES = 64 * 1024 + + +def read_singlefile_header(path: pathlib.Path) -> Optional[str]: + """Return the start of a SingleFile, through the end of its header comment, or None if the header comment + does not end within SINGLEFILE_HEADER_MAX_BYTES. A partial header could cut off its URL or saved date.""" + head = b'' + with path.open('rb') as fh: + # Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail. + while len(head) < SINGLEFILE_HEADER_MAX_BYTES and (chunk := fh.read(1000)): + head += chunk + if b'-->' in head: + return head.decode(errors='ignore') + return None + + +def parse_singlefile_saved_date(saved_date: str) -> Optional[datetime]: + """Parse SingleFile's "saved date", which is the browser's local time and its offset, into UTC. + + Returns None if the offset is malformed; the instant is unknown. + + >>> parse_singlefile_saved_date('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)') + datetime.datetime(2024, 11, 8, 1, 36, 23, tzinfo=) + """ + parts = saved_date.split(' ') + if len(parts) < 6: + # No offset; assume GMT. + return dates.strpdate(' '.join(parts[:5])).replace(tzinfo=pytz.timezone('GMT')) + try: + return datetime.strptime(' '.join(parts[:6]), '%a %b %d %Y %H:%M:%S GMT%z').astimezone(pytz.UTC) + except ValueError: + return None + + class Archive(Base, ModelHelper): __tablename__ = 'archive' __table_args__ = ( @@ -314,30 +350,24 @@ class Archive(Base, ModelHelper): # This data has already been read. return - with path.open('rb') as fh: - # Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail. - head = fh.read(1000).decode(errors='ignore') - if 'Page saved with SingleFile' not in head: - logger.error(f'Could not find SingleFile header in {self}') - return + head = read_singlefile_header(path) + if not head or 'Page saved with SingleFile' not in head: + logger.error(f'Could not find SingleFile header in {self}') + return - if not self.file_group.url: - try: - if match := MATCH_URL.findall(head): - self.file_group.url = match[0].strip() - except Exception as e: - logger.error(f'Could not get URL from singlefile {path}', exc_info=e) + if not self.file_group.url: + try: + if match := MATCH_URL.findall(head): + self.file_group.url = match[0].strip() + except Exception as e: + logger.error(f'Could not get URL from singlefile {path}', exc_info=e) - if not self.file_group.download_datetime: - try: - if match := MATCH_DATE.findall(head): - dt = match[0].strip() - dt = ' '.join(dt.split(' ')[:5]) - # SingleFile uses GMT. - dt = dates.strpdate(dt).replace(tzinfo=pytz.timezone('GMT')) - self.file_group.download_datetime = dt - except Exception as e: - logger.error(f'Could not get archive date from singlefile {path}', exc_info=e) + if not self.file_group.download_datetime: + try: + if match := MATCH_DATE.findall(head): + self.file_group.download_datetime = parse_singlefile_saved_date(match[0].strip()) + except Exception as e: + logger.error(f'Could not get archive date from singlefile {path}', exc_info=e) def apply_domain(self): """Get the domain collection from the URL.""" diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 0a49dee3..316f58b1 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -2,7 +2,7 @@ import json import pathlib import re import tempfile -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from http import HTTPStatus from unittest import mock @@ -633,6 +633,131 @@ async def test_refresh_archives_index(test_session, make_files_structure, refres assert archive.file_group.indexed is True +@pytest.mark.asyncio +@pytest.mark.parametrize('saved_date,expected', [ + ('Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)', datetime(2022, 5, 16, 23, 51, 35)), + # The browser writes its local time; the offset must be applied, not dropped. + ('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)', datetime(2024, 11, 8, 1, 36, 23)), + ('Tue Mar 03 2026 09:15:00 GMT+0200 (Eastern European Standard Time)', datetime(2026, 3, 3, 7, 15, 0)), +]) +async def test_singlefile_saved_date_timezone(test_session, make_files_structure, refresh_files, saved_date, expected): + """An Archive's download_datetime is the SingleFile "saved date" converted to UTC.""" + make_files_structure({ + 'archive/example.com/NA.html': f''' ''', + }) + + await refresh_files() + + archive: Archive = test_session.query(Archive).one() + assert archive.file_group.download_datetime == expected.replace(tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_singlefile_saved_date_after_long_url(test_session, make_files_structure, refresh_files): + """The whole SingleFile header is read, so a long URL does not cut the saved date's offset off.""" + url = 'https://example.com/' + 'a' * 900 + make_files_structure({ + 'archive/example.com/NA.html': f''' ''', + }) + + await refresh_files() + + archive: Archive = test_session.query(Archive).one() + assert archive.file_group.url == url + assert archive.file_group.download_datetime == datetime(2025, 5, 26, 22, 10, 33, tzinfo=timezone.utc) + + +@pytest.mark.parametrize('saved_date,expected', [ + ('Mon May 26 2025 16:10:33 GMT-0600 (Mountain Daylight Time)', datetime(2025, 5, 26, 22, 10, 33)), + ('Mon May 26 2025 16:10:33 GMT+05:30', datetime(2025, 5, 26, 10, 40, 33)), + # No offset at all is GMT. + ('Mon May 26 2025 16:10:33', datetime(2025, 5, 26, 16, 10, 33)), + # A cut-off offset is unknown, not GMT. + ('Mon May 26 2025 16:10:33 GMT-06', None), +]) +def test_parse_singlefile_saved_date(saved_date, expected): + from modules.archive.models import parse_singlefile_saved_date + expected = expected.replace(tzinfo=timezone.utc) if expected else None + assert parse_singlefile_saved_date(saved_date) == expected + + +@pytest.mark.asyncio +async def test_repair_singlefile_saved_dates_unreadable(test_session, make_files_structure, refresh_files): + """The repair is not marked done while an Archive could not be read, so it runs again.""" + misread = datetime(2024, 11, 7, 18, 36, 23, tzinfo=timezone.utc) + singlefile, = make_files_structure({ + 'archive/example.com/NA.html': ''' ''', + }) + await refresh_files() + test_session.query(Archive).one().file_group.download_datetime = misread + test_session.commit() + + singlefile.chmod(0o000) + try: + assert lib.repair_singlefile_saved_dates() == 0 + finally: + singlefile.chmod(0o644) + + assert lib.repair_singlefile_saved_dates() == 1 + test_session.expire_all() + assert test_session.query(Archive).one().file_group.download_datetime == \ + datetime(2024, 11, 8, 1, 36, 23, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_repair_singlefile_saved_dates(test_session, make_files_structure, refresh_files): + """An Archive whose download_datetime is its SingleFile saved date's local time read as UTC is corrected, once. + Any other download_datetime is never changed.""" + mountain = 'Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)' + make_files_structure({ + f'archive/example.com/{name}.html': f''' ''' + for name, saved_date in (('misread', mountain), ('downloaded', mountain), + ('utc', 'Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)')) + }) + await refresh_files() + + def file_groups() -> dict: + test_session.expire_all() + return {i.file_group.primary_path.stem: i.file_group for i in test_session.query(Archive)} + + # What the offset-dropping parser stored, and a time that came from elsewhere (e.g. WROLPi's own download). + misread = datetime(2024, 11, 7, 18, 36, 23, tzinfo=timezone.utc) + downloaded = datetime(2024, 11, 8, 1, 36, 50, tzinfo=timezone.utc) + fgs = file_groups() + fgs['misread'].download_datetime = misread + fgs['downloaded'].download_datetime = downloaded + test_session.commit() + + assert lib.repair_singlefile_saved_dates() == 1 + + fgs = file_groups() + assert fgs['misread'].download_datetime == datetime(2024, 11, 8, 1, 36, 23, tzinfo=timezone.utc) + assert fgs['downloaded'].download_datetime == downloaded + assert fgs['utc'].download_datetime == datetime(2022, 5, 16, 23, 51, 35, tzinfo=timezone.utc) + + # The repair is done; it does not run again. + fgs['misread'].download_datetime = misread + test_session.commit() + assert lib.repair_singlefile_saved_dates() == 0 + assert file_groups()['misread'].download_datetime == misread + + @pytest.mark.asyncio async def test_archive_meta(async_client, test_session, make_files_structure, refresh_files): # The start of a typical singlefile html file. diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 2bebcc8c..da05b252 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -17,7 +17,7 @@ import urllib.parse import zipfile from itertools import zip_longest from pathlib import Path -from typing import Callable, List, Tuple, Union, Dict, Generator, Iterable, Set +from typing import Callable, List, Optional, Tuple, Union, Dict, Generator, Iterable, Set import cachetools.func from sqlalchemy import asc, or_, text as sa_text @@ -59,7 +59,7 @@ __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories', 'get_normalized_ignored_directories', 'sanitize_ignored_directories', 'cached_search_total', 'search_filter_cache_key', 'count_file_groups', - 'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems', + 'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems', 'get_kv_value', 'set_kv_value', 'backfill_null_file_group_stems', 'dedupe_file_groups_by_directory_stem', 'merge_file_group_relations', 'merge_into_surviving_duplicates'] @@ -913,25 +913,35 @@ def _ensure_kv_table(curs): curs.execute('CREATE TABLE IF NOT EXISTS wrolpi_kv (key TEXT PRIMARY KEY, value TEXT NOT NULL)') -def get_stem_algorithm_version() -> int: - """The grouping-algorithm version last applied to file_group.stem, or 0 if never stored.""" +def get_kv_value(key: str) -> Optional[str]: + """The value stored in wrolpi_kv for `key`, or None.""" with get_db_curs() as curs: _ensure_kv_table(curs) - curs.execute('SELECT value FROM wrolpi_kv WHERE key = ?', (STEM_ALGORITHM_VERSION_KEY,)) + curs.execute('SELECT value FROM wrolpi_kv WHERE key = ?', (key,)) row = curs.fetchone() - return int(row['value']) if row else 0 + return row['value'] if row else None -def set_stem_algorithm_version(version: int): +def set_kv_value(key: str, value: str): with get_db_curs(commit=True) as curs: _ensure_kv_table(curs) curs.execute( 'INSERT INTO wrolpi_kv (key, value) VALUES (?, ?) ' 'ON CONFLICT(key) DO UPDATE SET value = excluded.value', - (STEM_ALGORITHM_VERSION_KEY, str(version)), + (key, value), ) +def get_stem_algorithm_version() -> int: + """The grouping-algorithm version last applied to file_group.stem, or 0 if never stored.""" + value = get_kv_value(STEM_ALGORITHM_VERSION_KEY) + return int(value) if value else 0 + + +def set_stem_algorithm_version(version: int): + set_kv_value(STEM_ALGORITHM_VERSION_KEY, str(version)) + + def ensure_file_group_stems(): """Recompute file_group.stem when STEM_ALGORITHM_VERSION has changed.