diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 57c44bbd..088497cc 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -283,6 +283,7 @@ async def archive_modeler(): except InvalidArchive: # It was not a real Archive. Many HTML files will not be an Archive. file_group.indexed = False + invalid_archives.add(file_group.id) session.commit() diff --git a/wrolpi/common.py b/wrolpi/common.py index 588f8c50..3aba4c37 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -9,6 +9,7 @@ import multiprocessing import os import pathlib import re +import shutil import string import sys import tempfile @@ -290,6 +291,15 @@ class ConfigFile: with config_file.open('rt') as fh: self._config.update(yaml.load(fh, Loader=yaml.Loader)) + def _get_backup_filename(self): + """Returns the path for the backup file for today.""" + # TODO what if the RPi clock is not working? Change this so the "version" of the config is used each day. + path = get_media_directory() / f'config/backup/{self.file_name}' + date_str = now().strftime('%Y%m%d') + name = f'{path.stem}-{date_str}{path.suffix}' + path = path.with_name(name) + return path + def save(self): """ Write this config to its file. @@ -310,15 +320,22 @@ class ConfigFile: if not config_file.parent.is_dir(): config_file.parent.mkdir() + backup_file = self._get_backup_filename() + # Read the existing config, replace all values, then save. if config_file.is_file(): with config_file.open('rt') as fh: config = yaml.load(fh, Loader=yaml.Loader) + # Copy the existing config to the backup directory. One for each day. + if not backup_file.parent.exists(): + backup_file.parent.mkdir(parents=True) + shutil.copy(config_file, backup_file) else: # Config file does not yet exist. config = dict() config.update({k: v for k, v in self._config.items() if v is not None}) + logger.warning(f'Saving config: {config_file}') with config_file.open('wt') as fh: yaml.dump(config, fh, width=self.width) finally: diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index 0dfd3d79..cefb723c 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -655,7 +655,9 @@ async def test_ignore_directory(test_session, test_async_client, test_directory, await lib.refresh_files() files = test_session.query(FileGroup).order_by(FileGroup.primary_path).all() - assert {i.primary_path.name for i in files} == {'foo.txt', 'bar.txt', 'baz.txt', 'wrolpi.yaml'} + # Ignore configs. + files = {i for i in files if not i.primary_path.name.endswith('.yaml')} + assert {i.primary_path.name for i in files} == {'foo.txt', 'bar.txt', 'baz.txt'} # Cannot ignore special directories. content = dict(path=str(test_directory / 'videos')) diff --git a/wrolpi/tags.py b/wrolpi/tags.py index ffc9fa38..47a5e01b 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -158,6 +158,7 @@ class TagsConfig(ConfigFile): tag_file.created_at.isoformat() if tag_file.created_at else dates.now().isoformat(), ] tag_files.append(value) + logger.debug(f'Got {len(tag_files)} tag files for config.') from modules.zim.models import Zim, TagZimEntry results = session.query(Tag, Zim, TagZimEntry) \ @@ -176,6 +177,7 @@ class TagsConfig(ConfigFile): tag_zim_entry.created_at.isoformat() if tag_zim_entry.created_at else dates.now().isoformat(), ] tag_zims.append(value) + logger.debug(f'Got {len(tag_zims)} tag zims for config.') # Write to the config. self.update({ @@ -481,3 +483,11 @@ def tag_names_to_zim_sub_select(tag_names: List[str], zim_id: int = None) -> Tup HAVING array_agg(t.name)::TEXT[] @> %(tag_names)s::TEXT[] ''' return stmt, params + + +if __name__ == '__main__': + from wrolpi.db import get_db_session + + if input('Save Tags to config file? (y/n) ').strip().lower() == 'y': + with get_db_session() as session: + get_tags_config().save_tags(session) diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 5e7a25da..98e70211 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -4,6 +4,7 @@ import json import multiprocessing import os import pathlib +import re import tempfile from datetime import date, datetime from decimal import Decimal @@ -797,3 +798,24 @@ def test_find_file(test_directory, make_files_structure): assert common.find_file(test_directory, 'baz.txt', 2) == baz assert common.find_file(test_directory, 'does not exist', 100) is None + + +def test_config_backup(test_config, test_directory): + """Configs have a backup each day.""" + config = common.get_wrolpi_config() + + # No backups yet. + config.save() + assert (test_directory / 'config').is_dir() + assert not (test_directory / 'config/backup').is_dir() + assert (test_directory / 'config/wrolpi.yaml').is_file() + + # Backups directory is created. New backup config is saved. + config.save() + assert (test_directory / 'config').is_dir() + assert (test_directory / 'config/backup').is_dir() + assert (test_directory / 'config/wrolpi.yaml').is_file() + + backup_file, = list((test_directory / 'config/backup').glob('*')) + assert re.match(r'wrolpi-\d{8}\.yaml', backup_file.name), \ + f'{backup_file.name} is not in the expected format'