mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-25 18:40:06 -04:00
Config versions count only the columns each config saves
A Repo's fetch (head_sha, last_fetch), a Channel's refresh (video_count, info_json), a Collection's counts, and a Download attempt (attempts, error) counted as config changes, so the next import of that config saved instead of importing: a hand edit to repos.yaml was overwritten after any fetch. Each model now registers only the columns its config saves. domains.yaml saves its domains' Downloads, which now count for it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
74e9ca7799
commit
d303cab06c
5 changed files with 77 additions and 9 deletions
|
|
@ -1562,7 +1562,9 @@ class ReposConfig(ConfigFile):
|
|||
|
||||
|
||||
repos_config = ReposConfig()
|
||||
register_ownership(ReposConfig.file_name, Repository)
|
||||
# Only the columns repos.yaml saves (not a fetch's commit, size, ...); its Collection saves the rest.
|
||||
register_ownership(ReposConfig.file_name, Repository,
|
||||
columns={'url', 'mode', 'branch', 'submodules', 'clone_token', 'collection_id'})
|
||||
|
||||
|
||||
def get_repos_config() -> ReposConfig:
|
||||
|
|
|
|||
|
|
@ -530,7 +530,10 @@ class ChannelsConfig(ConfigFile):
|
|||
|
||||
|
||||
CHANNELS_CONFIG: ChannelsConfig = ChannelsConfig()
|
||||
register_ownership(ChannelsConfig.file_name, Channel)
|
||||
# Only the columns channels.yaml saves (not a refresh's counts, info_json, ...); its Collection saves the rest.
|
||||
register_ownership(ChannelsConfig.file_name, Channel,
|
||||
columns={'url', 'source_id', 'generate_posters', 'calculate_duration', 'download_missing_data',
|
||||
'collection_id'})
|
||||
TEST_CHANNELS_CONFIG: ChannelsConfig = None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1316,9 +1316,11 @@ class CollectionItem(ModelHelper, Base):
|
|||
return data
|
||||
|
||||
|
||||
# Each kind of Collection is saved by its own config.
|
||||
# Each kind of Collection is saved by its own config, which saves these columns (not item_count, total_size, ...).
|
||||
COLLECTION_CONFIG_COLUMNS = {'name', 'kind', 'description', 'directory', 'tag_id', 'file_format'}
|
||||
for _kind, _config in (('channel', 'channels.yaml'), ('domain', 'domains.yaml'), ('playlist', 'playlists.yaml'),
|
||||
('repo', 'repos.yaml')):
|
||||
register_ownership(_config, Collection, row=lambda collection, kind=_kind: collection.kind == kind)
|
||||
register_ownership(_config, Collection, columns=COLLECTION_CONFIG_COLUMNS,
|
||||
row=lambda collection, kind=_kind: collection.kind == kind)
|
||||
# Only a playlist's items are saved (author and subject Collections have items too).
|
||||
register_ownership('playlists.yaml', CollectionItem, row=lambda item: 'playlist' in collection_kinds(item))
|
||||
|
|
|
|||
|
|
@ -2310,11 +2310,14 @@ class DownloadManagerConfig(ConfigFile):
|
|||
|
||||
|
||||
DOWNLOAD_MANAGER_CONFIG: DownloadManagerConfig = DownloadManagerConfig()
|
||||
register_ownership(DownloadManagerConfig.file_name, Download)
|
||||
# channels.yaml and repos.yaml each save their own Collections' Downloads (the URL and how often it runs). Only
|
||||
# the config of that Collection's kind owns one: counting another config's change would make that config's next
|
||||
# import save instead of import.
|
||||
DOWNLOAD_COLLECTION_CONFIGS = (('channel', 'channels.yaml'), ('repo', 'repos.yaml'))
|
||||
# Each config owns only the columns it saves: counting another change would make its next import save instead of
|
||||
# import. (`attempts`, `error`, ... change on every attempt.)
|
||||
register_ownership(DownloadManagerConfig.file_name, Download,
|
||||
columns={'destination', 'downloader', 'frequency', 'last_successful_download', 'next_download',
|
||||
'settings', 'status', 'sub_downloader', 'tag_names', 'url'})
|
||||
# channels.yaml, domains.yaml and repos.yaml each save their own Collections' Downloads (the URL and how often it
|
||||
# runs). Only the config of that Collection's kind owns one.
|
||||
DOWNLOAD_COLLECTION_CONFIGS = (('channel', 'channels.yaml'), ('domain', 'domains.yaml'), ('repo', 'repos.yaml'))
|
||||
for _kind, _config in DOWNLOAD_COLLECTION_CONFIGS:
|
||||
register_ownership(_config, Download, columns={'url', 'frequency', 'collection_id'},
|
||||
row=lambda download, kind=_kind: kind in collection_kinds(download))
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import yaml
|
|||
|
||||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.config_version import get_config_version, set_config_version
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.tags import Tag, get_tags_config, import_tags_config
|
||||
|
||||
|
|
@ -276,3 +277,60 @@ def test_only_playlist_items_count_for_playlists(test_session):
|
|||
playlist.add_url(test_session, '/map?x=1', title='Spot')
|
||||
test_session.commit()
|
||||
assert db_version(test_session, 'playlists.yaml') == 2
|
||||
|
||||
|
||||
def test_unsaved_columns_do_not_count(test_session):
|
||||
"""Columns a config does not save (a fetch's commit, a refresh's counts, a download attempt) do not count:
|
||||
counting them would make that config's next import save instead of import."""
|
||||
from modules.repos.models import Repository
|
||||
from modules.videos.models import Channel
|
||||
from wrolpi.downloader import Download
|
||||
|
||||
channel_collection = Collection(name='Channel', kind='channel')
|
||||
repo_collection = Collection(name='Repo', kind='repo')
|
||||
test_session.add_all([channel_collection, repo_collection])
|
||||
test_session.flush()
|
||||
channel = Channel(collection_id=channel_collection.id)
|
||||
repo = Repository(url='https://example.com/repo.git', collection_id=repo_collection.id)
|
||||
download = Download(url='https://example.com/channel', frequency=86400, collection_id=channel_collection.id)
|
||||
test_session.add_all([channel, repo, download])
|
||||
test_session.commit()
|
||||
names = ('channels.yaml', 'repos.yaml', 'download_manager.yaml')
|
||||
for name in names:
|
||||
set_config_version(test_session, name, 1)
|
||||
test_session.commit()
|
||||
|
||||
repo.head_sha = 'abc123'
|
||||
repo.last_fetch = now()
|
||||
channel.video_count = 10
|
||||
channel.refreshed = True
|
||||
channel_collection.item_count = 10
|
||||
repo_collection.total_size = 1000
|
||||
download.attempts = 1
|
||||
download.error = 'Failed'
|
||||
download.last_download_attempt = now()
|
||||
test_session.commit()
|
||||
assert {i: db_version(test_session, i) for i in names} == dict.fromkeys(names, 1)
|
||||
|
||||
# Saved columns still count.
|
||||
repo.branch = 'dev'
|
||||
channel.generate_posters = True
|
||||
download.status = 'failed'
|
||||
test_session.commit()
|
||||
assert {i: db_version(test_session, i) for i in names} == dict.fromkeys(names, 2)
|
||||
|
||||
|
||||
def test_domain_downloads_count_for_domains(test_session):
|
||||
"""domains.yaml saves each domain's Downloads (url, frequency)."""
|
||||
from wrolpi.downloader import Download
|
||||
|
||||
domain = Collection(name='example.com', kind='domain')
|
||||
test_session.add(domain)
|
||||
test_session.commit()
|
||||
set_config_version(test_session, 'domains.yaml', 1)
|
||||
test_session.commit()
|
||||
|
||||
test_session.add(Download(url='https://example.com/feed', frequency=86400, collection_id=domain.id))
|
||||
test_session.commit()
|
||||
|
||||
assert db_version(test_session, 'domains.yaml') == 2
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue