rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-25 18:40:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Apply the offset in a SingleFile saved date, and repair misread Archive dates (#862)

Browse source 
* Apply the offset in a SingleFile saved date

SingleFile writes the browser's local time with its offset ("... 18:36:23 GMT-0700").  The parser dropped
the offset and treated the local time as UTC, so an uploaded Archive's download_datetime was off by the
browser's UTC offset, and a Collection reorganize renamed its files with that shifted timestamp.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Repair Archive download datetimes misread from a SingleFile saved date

Archives parsed before the offset fix keep the shifted download_datetime, because the parser only fills
an empty value.  A one-time background repair (gated by a wrolpi_kv version, after the first refresh)
re-reads each Archive's saved date and corrects a download_datetime that equals the offset-dropped value.
Any other download_datetime is left alone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Read the whole SingleFile header; finish the date repair only when every Archive was read

The header was read in 1000 bytes, so a long URL cut the saved date's offset off, and the cut-off date was
stored as GMT.  The header is now read through the end of its comment, and a malformed offset leaves the
date unset instead of guessing.

The repair is no longer marked done when an Archive could not be read (a missing file is a deleted Archive,
not a failure), and it waits for the media to be mounted, because `refresh_complete` is restored from the
database before the drives are checked.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-10-23 14:34:29 -06:00 • committed by GitHub
parent 1f3762ff20
commit da3dadde07
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 275 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

17
main.py
View file

@ -423,6 +423,23 @@ async def perpetual_start_video_missing_comments_download():
logger.debug('Waiting for downloads to be enabled before downloading comments...')
@perpetual_signal(sleep=3600)
async def perpetual_repair_archive_saved_dates():
"""Correct Archive download datetimes that were misread from their SingleFile saved date. Runs once."""
async with flags.refresh_complete.wait_for():
# Archives must be in the database.
pass
async with flags.media_mounted.wait_for():
# Their files must be readable; `refresh_complete` is restored from the database before the drive is checked.
pass
from modules.archive.lib import repair_singlefile_saved_dates
try:
await asyncio.to_thread(repair_singlefile_saved_dates)
except Exception as e:
logger.error('Failed to repair Archive download datetimes', exc_info=e)
@perpetual_signal(sleep=3600) # Check hourly
async def perpetual_check_for_updates():
"""

65
modules/archive/lib.py
View file

@ -19,20 +19,20 @@ import pytz
from sqlalchemy import asc, func, BigInteger
from sqlalchemy.orm import Session
from modules.archive.models import Archive
from modules.archive.models import Archive, MATCH_DATE, parse_singlefile_saved_date, read_singlefile_header
from wrolpi import dates
from wrolpi import fts
from wrolpi.cmd import READABILITY_BIN, run_command
from wrolpi.collections import Collection
from wrolpi.common import get_media_directory, get_relative_to_media_directory, logger, extract_domain, \
escape_file_name, aiohttp_post, format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, \
get_wrolpi_config, html_screenshot, html_file_screenshot, ConfigFile, trim_file_name
get_wrolpi_config, html_screenshot, html_file_screenshot, ConfigFile, trim_file_name, chunks
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, session_curs
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
from wrolpi.events import Events
from wrolpi.files.lib import handle_file_group_search_results, cached_search_total, \
search_filter_cache_key, count_file_groups
search_filter_cache_key, count_file_groups, get_kv_value, set_kv_value
from wrolpi.files.models import FileGroup
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.tags import tag_append_sub_select_where
@ -1849,3 +1849,62 @@ async def generate_archive_screenshot(archive_id: int) -> pathlib.Path:
session.flush()
return screenshot_path
# Bump to run `repair_singlefile_saved_dates` again.
SAVED_DATE_REPAIR_VERSION = '1'
SAVED_DATE_REPAIR_KEY = 'archive_saved_date_repair_version'
def repair_singlefile_saved_dates() -> int:
"""Correct each Archive whose download_datetime is its SingleFile "saved date" local time read as UTC.
The saved date was once parsed without its offset. Only a download_datetime equal to that misread value, from a
saved date with a non-zero offset, is changed; any other download_datetime is never touched. Runs once per
SAVED_DATE_REPAIR_VERSION, and is not marked done while any Archive could not be read. A missing file is a
deleted Archive, not a failure. Returns the number of Archives corrected."""
if get_kv_value(SAVED_DATE_REPAIR_KEY) == SAVED_DATE_REPAIR_VERSION:
return 0
with get_db_session() as session:
rows = session.query(FileGroup.id, FileGroup.primary_path, FileGroup.download_datetime) \
.join(Archive, Archive.file_group_id == FileGroup.id) \
.filter(FileGroup.download_datetime.isnot(None)) \
.all()
corrections = dict()
missing = failed = 0
for file_group_id, path, download_datetime in rows:
try:
head = read_singlefile_header(path)
except FileNotFoundError:
missing += 1
continue
except OSError as e:
logger.warning(f'Could not read the SingleFile header of {path}', exc_info=e)
failed += 1
continue
if not head or not (match := MATCH_DATE.search(head)):
continue
saved_date = match.group(1).strip()
try:
corrected = parse_singlefile_saved_date(saved_date)
# The value the offset-dropping parser stored.
misread = dates.strpdate(' '.join(saved_date.split(' ')[:5])).replace(tzinfo=pytz.timezone('GMT'))
except Exception:
continue
if corrected and corrected != misread and download_datetime == misread:
corrections[file_group_id] = corrected
for chunk in chunks(list(corrections.items()), 500):
with get_db_session(commit=True) as session:
for file_group in session.query(FileGroup).filter(FileGroup.id.in_([i for i, _ in chunk])):
file_group.download_datetime = corrections[file_group.id]
logger.info(f'Corrected the download datetime of {len(corrections)} Archives from their SingleFile saved date'
f' ({missing} missing)')
if failed:
logger.warning(f'Could not read {failed} Archives; their download datetimes will be repaired later')
else:
set_kv_value(SAVED_DATE_REPAIR_KEY, SAVED_DATE_REPAIR_VERSION)
return len(corrections)

74
modules/archive/models.py
View file

@ -1,6 +1,7 @@
import json
import pathlib
import re
from datetime import datetime
from typing import Iterable, List, Optional
import pytz
@ -27,6 +28,41 @@ MATCH_URL = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE)
MATCH_DATE = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE)
# A SingleFile header holds the page's URL, which may be very long.
SINGLEFILE_HEADER_MAX_BYTES = 64 * 1024
def read_singlefile_header(path: pathlib.Path) -> Optional[str]:
"""Return the start of a SingleFile, through the end of its header comment, or None if the header comment
does not end within SINGLEFILE_HEADER_MAX_BYTES. A partial header could cut off its URL or saved date."""
head = b''
with path.open('rb') as fh:
# Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail.
while len(head) < SINGLEFILE_HEADER_MAX_BYTES and (chunk := fh.read(1000)):
head += chunk
if b'-->' in head:
return head.decode(errors='ignore')
return None
def parse_singlefile_saved_date(saved_date: str) -> Optional[datetime]:
"""Parse SingleFile's "saved date", which is the browser's local time and its offset, into UTC.
Returns None if the offset is malformed; the instant is unknown.
>>> parse_singlefile_saved_date('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)')
datetime.datetime(2024, 11, 8, 1, 36, 23, tzinfo=<UTC>)
"""
parts = saved_date.split(' ')
if len(parts) < 6:
# No offset; assume GMT.
return dates.strpdate(' '.join(parts[:5])).replace(tzinfo=pytz.timezone('GMT'))
try:
return datetime.strptime(' '.join(parts[:6]), '%a %b %d %Y %H:%M:%S GMT%z').astimezone(pytz.UTC)
except ValueError:
return None
class Archive(Base, ModelHelper):
__tablename__ = 'archive'
__table_args__ = (
@ -314,30 +350,24 @@ class Archive(Base, ModelHelper):
# This data has already been read.
return
with path.open('rb') as fh:
# Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail.
head = fh.read(1000).decode(errors='ignore')
if 'Page saved with SingleFile' not in head:
logger.error(f'Could not find SingleFile header in {self}')
return
head = read_singlefile_header(path)
if not head or 'Page saved with SingleFile' not in head:
logger.error(f'Could not find SingleFile header in {self}')
return
if not self.file_group.url:
try:
if match := MATCH_URL.findall(head):
self.file_group.url = match[0].strip()
except Exception as e:
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
if not self.file_group.url:
try:
if match := MATCH_URL.findall(head):
self.file_group.url = match[0].strip()
except Exception as e:
logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
if not self.file_group.download_datetime:
try:
if match := MATCH_DATE.findall(head):
dt = match[0].strip()
dt = ' '.join(dt.split(' ')[:5])
# SingleFile uses GMT.
dt = dates.strpdate(dt).replace(tzinfo=pytz.timezone('GMT'))
self.file_group.download_datetime = dt
except Exception as e:
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
if not self.file_group.download_datetime:
try:
if match := MATCH_DATE.findall(head):
self.file_group.download_datetime = parse_singlefile_saved_date(match[0].strip())
except Exception as e:
logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
def apply_domain(self):
"""Get the domain collection from the URL."""

127
modules/archive/test/test_lib.py
View file

@ -2,7 +2,7 @@ import json
import pathlib
import re
import tempfile
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from http import HTTPStatus
from unittest import mock
@ -633,6 +633,131 @@ async def test_refresh_archives_index(test_session, make_files_structure, refres
assert archive.file_group.indexed is True
@pytest.mark.asyncio
@pytest.mark.parametrize('saved_date,expected', [
('Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)', datetime(2022, 5, 16, 23, 51, 35)),
# The browser writes its local time; the offset must be applied, not dropped.
('Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)', datetime(2024, 11, 8, 1, 36, 23)),
('Tue Mar 03 2026 09:15:00 GMT+0200 (Eastern European Standard Time)', datetime(2026, 3, 3, 7, 15, 0)),
])
async def test_singlefile_saved_date_timezone(test_session, make_files_structure, refresh_files, saved_date, expected):
"""An Archive's download_datetime is the SingleFile "saved date" converted to UTC."""
make_files_structure({
'archive/example.com/NA.html': f'''<!DOCTYPE html> <html lang="en"><!--
Page saved with SingleFile
url: https://example.com
saved date: {saved_date}
--><head><meta charset="utf-8"></head></html>''',
})
await refresh_files()
archive: Archive = test_session.query(Archive).one()
assert archive.file_group.download_datetime == expected.replace(tzinfo=timezone.utc)
@pytest.mark.asyncio
async def test_singlefile_saved_date_after_long_url(test_session, make_files_structure, refresh_files):
"""The whole SingleFile header is read, so a long URL does not cut the saved date's offset off."""
url = 'https://example.com/' + 'a' * 900
make_files_structure({
'archive/example.com/NA.html': f'''<!DOCTYPE html> <html lang="en"><!--
Page saved with SingleFile
url: {url}
saved date: Mon May 26 2025 16:10:33 GMT-0600 (Mountain Daylight Time)
--><head><meta charset="utf-8"></head></html>''',
})
await refresh_files()
archive: Archive = test_session.query(Archive).one()
assert archive.file_group.url == url
assert archive.file_group.download_datetime == datetime(2025, 5, 26, 22, 10, 33, tzinfo=timezone.utc)
@pytest.mark.parametrize('saved_date,expected', [
('Mon May 26 2025 16:10:33 GMT-0600 (Mountain Daylight Time)', datetime(2025, 5, 26, 22, 10, 33)),
('Mon May 26 2025 16:10:33 GMT+05:30', datetime(2025, 5, 26, 10, 40, 33)),
# No offset at all is GMT.
('Mon May 26 2025 16:10:33', datetime(2025, 5, 26, 16, 10, 33)),
# A cut-off offset is unknown, not GMT.
('Mon May 26 2025 16:10:33 GMT-06', None),
])
def test_parse_singlefile_saved_date(saved_date, expected):
from modules.archive.models import parse_singlefile_saved_date
expected = expected.replace(tzinfo=timezone.utc) if expected else None
assert parse_singlefile_saved_date(saved_date) == expected
@pytest.mark.asyncio
async def test_repair_singlefile_saved_dates_unreadable(test_session, make_files_structure, refresh_files):
"""The repair is not marked done while an Archive could not be read, so it runs again."""
misread = datetime(2024, 11, 7, 18, 36, 23, tzinfo=timezone.utc)
singlefile, = make_files_structure({
'archive/example.com/NA.html': '''<!DOCTYPE html> <html lang="en"><!--
Page saved with SingleFile
url: https://example.com
saved date: Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)
--><head><meta charset="utf-8"></head></html>''',
})
await refresh_files()
test_session.query(Archive).one().file_group.download_datetime = misread
test_session.commit()
singlefile.chmod(0o000)
try:
assert lib.repair_singlefile_saved_dates() == 0
finally:
singlefile.chmod(0o644)
assert lib.repair_singlefile_saved_dates() == 1
test_session.expire_all()
assert test_session.query(Archive).one().file_group.download_datetime == \
datetime(2024, 11, 8, 1, 36, 23, tzinfo=timezone.utc)
@pytest.mark.asyncio
async def test_repair_singlefile_saved_dates(test_session, make_files_structure, refresh_files):
"""An Archive whose download_datetime is its SingleFile saved date's local time read as UTC is corrected, once.
Any other download_datetime is never changed."""
mountain = 'Thu Nov 07 2024 18:36:23 GMT-0700 (Mountain Standard Time)'
make_files_structure({
f'archive/example.com/{name}.html': f'''<!DOCTYPE html> <html lang="en"><!--
Page saved with SingleFile
url: https://example.com/{name}
saved date: {saved_date}
--><head><meta charset="utf-8"></head></html>'''
for name, saved_date in (('misread', mountain), ('downloaded', mountain),
('utc', 'Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time)'))
})
await refresh_files()
def file_groups() -> dict:
test_session.expire_all()
return {i.file_group.primary_path.stem: i.file_group for i in test_session.query(Archive)}
# What the offset-dropping parser stored, and a time that came from elsewhere (e.g. WROLPi's own download).
misread = datetime(2024, 11, 7, 18, 36, 23, tzinfo=timezone.utc)
downloaded = datetime(2024, 11, 8, 1, 36, 50, tzinfo=timezone.utc)
fgs = file_groups()
fgs['misread'].download_datetime = misread
fgs['downloaded'].download_datetime = downloaded
test_session.commit()
assert lib.repair_singlefile_saved_dates() == 1
fgs = file_groups()
assert fgs['misread'].download_datetime == datetime(2024, 11, 8, 1, 36, 23, tzinfo=timezone.utc)
assert fgs['downloaded'].download_datetime == downloaded
assert fgs['utc'].download_datetime == datetime(2022, 5, 16, 23, 51, 35, tzinfo=timezone.utc)
# The repair is done; it does not run again.
fgs['misread'].download_datetime = misread
test_session.commit()
assert lib.repair_singlefile_saved_dates() == 0
assert file_groups()['misread'].download_datetime == misread
@pytest.mark.asyncio
async def test_archive_meta(async_client, test_session, make_files_structure, refresh_files):
# The start of a typical singlefile html file.

26
wrolpi/files/lib.py
View file

@ -17,7 +17,7 @@ import urllib.parse
import zipfile
from itertools import zip_longest
from pathlib import Path
from typing import Callable, List, Tuple, Union, Dict, Generator, Iterable, Set
from typing import Callable, List, Optional, Tuple, Union, Dict, Generator, Iterable, Set
import cachetools.func
from sqlalchemy import asc, or_, text as sa_text
@ -59,7 +59,7 @@ __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix',
'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories',
'get_normalized_ignored_directories', 'sanitize_ignored_directories',
'cached_search_total', 'search_filter_cache_key', 'count_file_groups',
'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems',
'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems', 'get_kv_value', 'set_kv_value',
'backfill_null_file_group_stems', 'dedupe_file_groups_by_directory_stem', 'merge_file_group_relations',
'merge_into_surviving_duplicates']
@ -913,25 +913,35 @@ def _ensure_kv_table(curs):
curs.execute('CREATE TABLE IF NOT EXISTS wrolpi_kv (key TEXT PRIMARY KEY, value TEXT NOT NULL)')
def get_stem_algorithm_version() -> int:
"""The grouping-algorithm version last applied to file_group.stem, or 0 if never stored."""
def get_kv_value(key: str) -> Optional[str]:
"""The value stored in wrolpi_kv for `key`, or None."""
with get_db_curs() as curs:
_ensure_kv_table(curs)
curs.execute('SELECT value FROM wrolpi_kv WHERE key = ?', (STEM_ALGORITHM_VERSION_KEY,))
curs.execute('SELECT value FROM wrolpi_kv WHERE key = ?', (key,))
row = curs.fetchone()
return int(row['value']) if row else 0
return row['value'] if row else None
def set_stem_algorithm_version(version: int):
def set_kv_value(key: str, value: str):
with get_db_curs(commit=True) as curs:
_ensure_kv_table(curs)
curs.execute(
'INSERT INTO wrolpi_kv (key, value) VALUES (?, ?) '
'ON CONFLICT(key) DO UPDATE SET value = excluded.value',
(STEM_ALGORITHM_VERSION_KEY, str(version)),
(key, value),
)
def get_stem_algorithm_version() -> int:
"""The grouping-algorithm version last applied to file_group.stem, or 0 if never stored."""
value = get_kv_value(STEM_ALGORITHM_VERSION_KEY)
return int(value) if value else 0
def set_stem_algorithm_version(version: int):
set_kv_value(STEM_ALGORITHM_VERSION_KEY, str(version))
def ensure_file_group_stems():
"""Recompute file_group.stem when STEM_ALGORITHM_VERSION has changed.