diff --git a/.circleci/config.yml b/.circleci/config.yml index 55b8366b..2c358b7b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,11 +8,6 @@ jobs: api-tests-3-11: docker: - image: cimg/python:3.11 - - image: cimg/postgres:15.6 - environment: - POSTGRES_USER: postgres - POSTGRES_DB: wrolpi - POSTGRES_PASSWORD: "wrolpi" # `large` (4 vCPU, 8 GB) so pytest-xdist's `-nauto` has room to scale. resource_class: large steps: @@ -41,11 +36,6 @@ jobs: api-alembic-migrations: docker: - image: cimg/python:3.11 - - image: cimg/postgres:15.6 - environment: - POSTGRES_USER: wrolpi - POSTGRES_DB: wrolpi - POSTGRES_PASSWORD: "wrolpi" resource_class: medium steps: - checkout @@ -57,24 +47,18 @@ jobs: python3 -m venv venv . venv/bin/activate pip install -r requirements.txt - - save_cache: - key: deps-3.11-{{ checksum "requirements.txt" }}-5 - paths: - - "venv" + - run: + name: Create Media Directory + command: mkdir -p /tmp/wrolpi-media/config - run: name: Run Alembic Migrations - command: ./venv/bin/python ./main.py db upgrade + command: MEDIA_DIRECTORY=/tmp/wrolpi-media ./venv/bin/python ./main.py db upgrade - run: name: Check for Missing Migrations - command: ./venv/bin/python ./main.py db check + command: MEDIA_DIRECTORY=/tmp/wrolpi-media ./venv/bin/python ./main.py db check api-tests-3-12: docker: - image: cimg/python:3.12 - - image: cimg/postgres:15.6 - environment: - POSTGRES_USER: postgres - POSTGRES_DB: wrolpi - POSTGRES_PASSWORD: "wrolpi" resource_class: large steps: - checkout diff --git a/.gitpod.yml b/.gitpod.yml index c7827355..0b74bb49 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -11,7 +11,6 @@ tasks: docker-compose build --parallel docker volume create --name=openstreetmap-data docker volume create --name=openstreetmap-rendered-tiles - docker-compose up -d db docker-compose run --rm api db upgrade # command runs each time a user starts their workspace command: docker-compose up diff --git a/README.md b/README.md index 3a191676..dfe580fd 100644 --- a/README.md +++ b/README.md @@ -79,13 +79,11 @@ You can try out WROLPi by running the docker containers. 6. Create volumes. * `docker volume create --name=openstreetmap-data` * `docker volume create --name=openstreetmap-rendered-tiles` -7. Start the database. - * `docker-compose up -d db` -8. Initialize the database. +7. Initialize the database (SQLite in the media directory). * `docker-compose run --rm api db upgrade` -9. Start the docker containers. +8. Start the docker containers. * `docker-compose up` -10. Browse to WROLPi. +9. Browse to WROLPi. * https://0.0.0.0:8443 More Docker documentation is available in [docker/README.md](docker/README.md) diff --git a/alembic.ini b/alembic.ini index c9d850c4..650ff0ae 100644 --- a/alembic.ini +++ b/alembic.ini @@ -35,11 +35,17 @@ prepend_sys_path = . # directories, initial revisions must be specified with --version-path # version_locations = %(here)s/bar %(here)s/bat alembic/versions +# Revision ids ARE dates (see process_revision_directives in env.py), so filenames self-sort +# chronologically and the alembic_version table says when a database's schema was created. +file_template = %%(rev)s_%%(slug)s + # the output encoding used when revision files # are written from script.py.mako # output_encoding = utf-8 -sqlalchemy.url = postgresql://postgres:wrolpi@localhost:5432/wrolpi +# Placeholder: env.py derives the real URL from the media directory (see wrolpi.db.get_db_uri); +# `main.py db ...` overrides it explicitly. +sqlalchemy.url = sqlite:///CHANGEME [post_write_hooks] diff --git a/alembic/env.py b/alembic/env.py index c00e553c..9aeeb116 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,9 +1,6 @@ -import os from logging.config import fileConfig from alembic import context -from sqlalchemy import engine_from_config -from sqlalchemy import pool # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -26,13 +23,40 @@ from wrolpi.collections.models import Collection, CollectionItem # noqa: F401 from modules.videos.models import Video, Channel # noqa: F401 from modules.archive.models import Archive # noqa: F401 from modules.zim.models import Zim, TagZimEntry, ZimSubscription # noqa: F401 + target_metadata = Base.metadata -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. +def get_url() -> str: + """The database URL: from the ini/CLI when overridden, otherwise derived from the media directory.""" + url = config.get_main_option('sqlalchemy.url') + if url and not url.startswith('sqlite:///CHANGEME'): + return url + from wrolpi.db import get_db_uri + return get_db_uri() + + +def include_object(object_, name, type_, reflected, compare_to): + """Exclude tables Alembic must not manage from autogenerate/check. + + The FTS5 virtual tables and their shadow tables (created by wrolpi.schema_ddl / + wrolpi.fts) plus SQLite internals would otherwise be proposed for DROP.""" + if type_ == 'table': + from wrolpi.fts import FTS_TABLE_PREFIXES + if name == 'sqlite_sequence' or any(name.startswith(prefix) for prefix in FTS_TABLE_PREFIXES): + return False + return True + + +def process_revision_directives(context_, revision, directives): + """New revisions get date ids (e.g. 2026_07_10_1350) instead of random hashes. + + Filenames therefore self-sort chronologically, and the `alembic_version` value in a + database says when its schema was created. Underscores, not dashes: alembic forbids + `-` in revision identifiers (it is reserved for relative-revision syntax like `head-1`).""" + from datetime import datetime, timezone + script = directives[0] + script.rev_id = datetime.now(timezone.utc).strftime('%Y_%m_%d_%H%M') def run_migrations_offline(): @@ -47,12 +71,14 @@ def run_migrations_offline(): script output. """ - url = config.get_main_option("sqlalchemy.url") context.configure( - url=url, + url=get_url(), target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}, + render_as_batch=True, + include_object=include_object, + process_revision_directives=process_revision_directives, ) with context.begin_transaction(): @@ -66,18 +92,18 @@ def run_migrations_online(): and associate a connection with the context. """ - connectable = engine_from_config( - config.get_section(config.config_ini_section), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) + from wrolpi.db import create_wrolpi_engine + + connectable = create_wrolpi_engine(get_url()) with connectable.connect() as connection: - if not os.environ.get('DOCKER', '').lower().startswith('t'): - # Set role when not in a docker container - connection.execute("SET ROLE 'wrolpi'") context.configure( - connection=connection, target_metadata=target_metadata + connection=connection, + target_metadata=target_metadata, + # SQLite's limited ALTER TABLE requires batch mode for most column changes. + render_as_batch=True, + include_object=include_object, + process_revision_directives=process_revision_directives, ) with context.begin_transaction(): diff --git a/alembic/script.py.mako b/alembic/script.py.mako index e13d8b06..2c015630 100644 --- a/alembic/script.py.mako +++ b/alembic/script.py.mako @@ -5,9 +5,8 @@ Revises: ${down_revision | comma,n} Create Date: ${create_date} """ -import os from alembic import op -from sqlalchemy.orm import Session +import sqlalchemy as sa ${imports if imports else ""} # revision identifiers, used by Alembic. @@ -16,18 +15,10 @@ down_revision = ${repr(down_revision)} branch_labels = ${repr(branch_labels)} depends_on = ${repr(depends_on)} -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - ${upgrades if upgrades else "pass"} - if not DOCKERIZED: - session.execute('ALTER TABLE public.TABLE_NAME OWNER TO wrolpi') - def downgrade(): ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/00f11f309c53_filegroup_censored.py b/alembic/versions/00f11f309c53_filegroup_censored.py deleted file mode 100644 index 82ebea74..00000000 --- a/alembic/versions/00f11f309c53_filegroup_censored.py +++ /dev/null @@ -1,33 +0,0 @@ -"""FileGroup.censored - -Revision ID: 00f11f309c53 -Revises: 1c041aeee0fc -Create Date: 2024-06-17 14:02:09.881477 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '00f11f309c53' -down_revision = '1c041aeee0fc' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE file_group ADD COLUMN censored BOOLEAN DEFAULT FALSE') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS censored') diff --git a/alembic/versions/0400005fe2dc_video_textsearch_coalesce_fix.py b/alembic/versions/0400005fe2dc_video_textsearch_coalesce_fix.py deleted file mode 100644 index a6b83c38..00000000 --- a/alembic/versions/0400005fe2dc_video_textsearch_coalesce_fix.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Video textsearch coalesce fix. - -Revision ID: 0400005fe2dc -Revises: 53f6780aa599 -Create Date: 2022-01-04 11:00:21.776124 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '0400005fe2dc' -down_revision = '53f6780aa599' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - # Replace the textsearch and index. The old textsearch failed if one of the columns was null. - session.execute('DROP INDEX IF EXISTS video_textsearch_idx') - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS textsearch') - session.execute(''' - ALTER TABLE video - ADD COLUMN textsearch tsvector - GENERATED ALWAYS AS ( - setweight(to_tsvector('english'::regconfig, COALESCE(title, '')), 'A') || - setweight(to_tsvector('english'::regconfig, COALESCE(caption, '')), 'D') - ) STORED - ''') - session.execute('CREATE INDEX video_textsearch_idx ON video USING GIN(textsearch)') - - -def downgrade(): - # The upgrade is backwards compatible. - pass diff --git a/alembic/versions/053f17f07c4e_init_inventory_tables.py b/alembic/versions/053f17f07c4e_init_inventory_tables.py deleted file mode 100644 index e31c7efa..00000000 --- a/alembic/versions/053f17f07c4e_init_inventory_tables.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Init inventory tables. - -Revision ID: 053f17f07c4e -Revises: f0d0086a73d1 -Create Date: 2021-09-20 16:21:00.322598 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -revision = '053f17f07c4e' -down_revision = 'f0d0086a73d1' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - CREATE TABLE inventories_version ( - version SERIAL PRIMARY KEY NOT NULL - ) - ''') - - session.execute(''' - CREATE TABLE public.inventory ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL UNIQUE, - viewed_at TIMESTAMP WITHOUT TIME ZONE, - created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - deleted_at TIMESTAMP WITHOUT TIME ZONE - )''') - - session.execute(''' - CREATE TABLE public.item ( - id SERIAL PRIMARY KEY, - inventory_id INTEGER REFERENCES inventory(id) ON DELETE CASCADE, - brand TEXT, - name TEXT, - count DECIMAL, - item_size DECIMAL, - unit TEXT, - serving INTEGER, - category TEXT, - subcategory TEXT, - expiration_date DATE, - purchase_date DATE, - created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - deleted_at TIMESTAMP WITHOUT TIME ZONE - )''') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.inventories_version OWNER TO wrolpi') - session.execute('ALTER TABLE public.inventory OWNER TO wrolpi') - session.execute('ALTER TABLE public.item OWNER TO wrolpi') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP TABLE IF EXISTS public.inventories_version''') - session.execute('DROP TABLE IF EXISTS public.item''') - session.execute('DROP TABLE IF EXISTS public.inventory''') diff --git a/alembic/versions/13542a230c78_archive_url_index.py b/alembic/versions/13542a230c78_archive_url_index.py deleted file mode 100644 index 5bd5c8ef..00000000 --- a/alembic/versions/13542a230c78_archive_url_index.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Archive url index. - -Revision ID: 13542a230c78 -Revises: c72d305cf11e -Create Date: 2022-11-02 15:19:42.408462 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '13542a230c78' -down_revision = 'c72d305cf11e' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('CREATE INDEX IF NOT EXISTS archive_url_idx ON archive(url)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP INDEX IF EXISTS archive_url_idx') diff --git a/alembic/versions/158973fda20f_lower_filegroup_primary_path.py b/alembic/versions/158973fda20f_lower_filegroup_primary_path.py deleted file mode 100644 index 850d12bc..00000000 --- a/alembic/versions/158973fda20f_lower_filegroup_primary_path.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Lower FileGroup.primary_path - -Revision ID: 158973fda20f -Revises: 69e9754d1a63 -Create Date: 2025-08-16 10:54:58.874094 - -""" -import os -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '158973fda20f' -down_revision = '69e9754d1a63' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - op.execute('CREATE INDEX file_group_primary_path_lower_idx ON file_group (LOWER(primary_path))') - - -def downgrade(): - op.execute('DROP INDEX IF EXISTS file_group_primary_path_lower_idx') diff --git a/alembic/versions/170d1be52bc7_download_destination.py b/alembic/versions/170d1be52bc7_download_destination.py deleted file mode 100644 index 8be969e1..00000000 --- a/alembic/versions/170d1be52bc7_download_destination.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Create Download.destination column. - -Revision ID: 170d1be52bc7 -Revises: 9ec4c765ef8d -Create Date: 2024-11-02 11:16:15.297850 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '170d1be52bc7' -down_revision = '9ec4c765ef8d' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS destination TEXT') - session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS tag_names TEXT[]') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE download DROP COLUMN IF EXISTS tag_names') - session.execute('ALTER TABLE download DROP COLUMN IF EXISTS destination') diff --git a/alembic/versions/184329ed5137_downloader.py b/alembic/versions/184329ed5137_downloader.py deleted file mode 100644 index c24fb200..00000000 --- a/alembic/versions/184329ed5137_downloader.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Downloader - -Revision ID: 184329ed5137 -Revises: 1870bec7c81e -Create Date: 2021-11-17 19:13:58.305140 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '184329ed5137' -down_revision = '1870bec7c81e' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - CREATE TABLE download ( - id SERIAL PRIMARY KEY, - url TEXT NOT NULL, - status TEXT DEFAULT 'new', - next_download TIMESTAMPTZ, - last_successful_download TIMESTAMPTZ, - frequency INT, - attempts INT DEFAULT 0 - ) - ''') - session.execute('ALTER TABLE video ADD COLUMN url TEXT') - session.execute('ALTER TABLE channel ADD COLUMN source_id TEXT') - session.execute('CREATE INDEX channel_source_id ON channel (source_id)') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.download OWNER TO wrolpi') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('DROP TABLE IF EXISTS download') - session.execute('DROP INDEX IF EXISTS channel_source_id') - session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS source_id') - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS url') diff --git a/alembic/versions/1870bec7c81e_initialize_archive_tables.py b/alembic/versions/1870bec7c81e_initialize_archive_tables.py deleted file mode 100644 index 2d1d4e41..00000000 --- a/alembic/versions/1870bec7c81e_initialize_archive_tables.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Initialize archive tables. - -Revision ID: 1870bec7c81e -Revises: 053f17f07c4e -Create Date: 2021-09-22 18:31:15.056697 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '1870bec7c81e' -down_revision = '053f17f07c4e' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - CREATE TABLE domains ( - id SERIAL PRIMARY KEY, - domain TEXT NOT NULL UNIQUE, - directory TEXT NOT NULL UNIQUE - )''') - - session.execute(''' - CREATE TABLE url ( - id SERIAL PRIMARY KEY, - domain_id INTEGER REFERENCES domains(id) NOT NULL, - url TEXT NOT NULL UNIQUE, - latest_datetime TIMESTAMPTZ - )''') - - session.execute(''' - CREATE TABLE archive ( - id SERIAL PRIMARY KEY, - url_id INTEGER REFERENCES url(id) ON DELETE CASCADE NOT NULL, - domain_id INTEGER REFERENCES domains(id) ON DELETE CASCADE NOT NULL, - singlefile_path TEXT, - readability_path TEXT, - readability_txt_path TEXT, - readability_json_path TEXT, - screenshot_path TEXT, - title TEXT, - archive_datetime TIMESTAMPTZ, - status TEXT - )''') - - session.execute('ALTER TABLE url ADD COLUMN latest_id INTEGER REFERENCES archive(id) ON DELETE CASCADE') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.domains OWNER TO wrolpi') - session.execute('ALTER TABLE public.url OWNER TO wrolpi') - session.execute('ALTER TABLE public.archive OWNER TO wrolpi') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP TABLE IF EXISTS domains CASCADE') - session.execute('DROP TABLE IF EXISTS url CASCADE') - session.execute('DROP TABLE IF EXISTS archive CASCADE') diff --git a/alembic/versions/1a70196c05bb_download_location.py b/alembic/versions/1a70196c05bb_download_location.py deleted file mode 100644 index 3172a9fb..00000000 --- a/alembic/versions/1a70196c05bb_download_location.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Download location - -Revision ID: 1a70196c05bb -Revises: d8fcdb7773c0 -Create Date: 2022-05-02 12:01:51.210070 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '1a70196c05bb' -down_revision = 'd8fcdb7773c0' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS location TEXT') - session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS error TEXT') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE download DROP COLUMN IF EXISTS location') - session.execute('ALTER TABLE download DROP COLUMN IF EXISTS error') diff --git a/alembic/versions/1c041aeee0fc_doc_indexer.py b/alembic/versions/1c041aeee0fc_doc_indexer.py deleted file mode 100644 index 718d891e..00000000 --- a/alembic/versions/1c041aeee0fc_doc_indexer.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Doc indexer. - -Revision ID: 1c041aeee0fc -Revises: d96a04eb2785 -Create Date: 2024-03-05 16:54:34.478974 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '1c041aeee0fc' -down_revision = 'd96a04eb2785' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - # Re-index Word documents. - bind = op.get_bind() - session = Session(bind=bind) - session.execute( - """ - UPDATE file_group - SET indexed=false - WHERE mimetype in - ('application/msword','application/vnd.openxmlformats-officedocument.wordprocessingml.document') - """) - - -def downgrade(): - pass diff --git a/alembic/versions/1f2325523525_download_destination_migrate.py b/alembic/versions/1f2325523525_download_destination_migrate.py deleted file mode 100644 index a1385d0c..00000000 --- a/alembic/versions/1f2325523525_download_destination_migrate.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Create `Download.destination` and `Download.tag_names`, migrate from `Download.settings`. - -Revision ID: 1f2325523525 -Revises: 170d1be52bc7 -Create Date: 2024-11-02 11:20:45.028865 - -""" -import os -from copy import copy - -from alembic import op -from sqlalchemy import Column, Integer, Text, String, ARRAY -from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Session - -from wrolpi.common import ModelHelper -from wrolpi.dates import TZDateTime -from wrolpi.downloader import get_download_manager_config, DownloadStatus - -# revision identifiers, used by Alembic. -revision = '1f2325523525' -down_revision = '170d1be52bc7' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - -Base = declarative_base() - - -class MDownload(ModelHelper, Base): - __tablename__ = 'download' - id = Column(Integer, primary_key=True) - url = Column(String, nullable=False, unique=True) - attempts = Column(Integer, default=0) - destination = Column(Text) - downloader = Column(Text) - sub_downloader = Column(Text) - error = Column(Text) - frequency = Column(Integer) - info_json = Column(JSONB) - last_successful_download = Column(TZDateTime) - location = Column(Text) - next_download = Column(TZDateTime) - settings = Column(JSONB) - status = Column(String, default=DownloadStatus.new) - tag_names = Column(ARRAY(Text)) - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - for download in session.query(MDownload): - if not download.settings: - continue - # Move `download.settings['destination']` to `download.destination`, if any. - settings = copy(download.settings) if download.settings else dict() - destination = settings.pop('destination', None) - tag_names = settings.pop('tag_names', None) - if isinstance(excluded_urls := settings.get('excluded_urls'), list): - settings['excluded_urls'] = ','.join(excluded_urls) - download.destination = destination or None - download.tag_names = tag_names or None - download.settings = settings - - # This will migrate the config as-is at the time that this migration was written. This code is probably not - # safe to reuse in the future! - - # Read config file directly. - try: - config = get_download_manager_config() - config._config.update(config.read_config_file()) - except FileNotFoundError: - # Config does not exist, probably testing, or a new WROLPi. - return - - new_downloads = [] - for download in session.query(MDownload).order_by(MDownload.url): - if download.last_successful_download and not download.frequency: - # This once-download has completed, do not save it. - continue - new_download = dict( - destination=download.destination, - downloader=download.downloader, - frequency=download.frequency, - last_successful_download=download.last_successful_download, - next_download=download.next_download, - settings=download.settings, - status=download.status, - sub_downloader=download.sub_downloader, - url=download.url, - ) - new_downloads.append(new_download) - - # Write directly to file, bypassing usual checks. - config._config['downloads'] = new_downloads - config.write_config_data(config._config, config.get_file()) - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - for download in session.query(MDownload): - if download.destination: - settings = copy(download.settings) if download.settings else dict() - settings['destination'] = download.destination or settings.get('destination') - settings['tag_names'] = download.tag_names or settings.get('tag_names') - if isinstance(excluded_urls := settings.get('excluded_urls'), str): - settings['excluded_urls'] = excluded_urls.split(',') - download.settings = settings - - try: - config = get_download_manager_config() - config._config.update(config.read_config_file()) - except FileNotFoundError: - # Config does not exist, probably testing, or a new WROLPi. - return - - new_downloads = [] - for download in session.query(MDownload).order_by(MDownload.url): - new_download = dict( - downloader=download.downloader, - frequency=download.frequency, - last_successful_download=download.last_successful_download, - next_download=download.next_download, - settings=download.settings, - status=download.status, - sub_downloader=download.sub_downloader, - url=download.url, - ) - new_downloads.append(new_download) - - config._config['downloads'] = new_downloads - config.write_config_data(config._config, config.get_file()) diff --git a/alembic/versions/2026_07_10_1350_sqlite_baseline.py b/alembic/versions/2026_07_10_1350_sqlite_baseline.py new file mode 100644 index 00000000..83648797 --- /dev/null +++ b/alembic/versions/2026_07_10_1350_sqlite_baseline.py @@ -0,0 +1,378 @@ +"""The SQLite baseline. + +The entire Postgres-era migration history (85 revisions) was squashed into this baseline when +WROLPi moved its database into the media directory (SQLite). See the git history of +alembic/versions/ for the old chain. + +Revision ID: 2026_07_10_1350 +Revises: +Create Date: 2026-07-10 13:50:59.070646 + +""" +from alembic import op +import sqlalchemy as sa + +import wrolpi.dates +import wrolpi.files.models +import wrolpi.media_path + + +# revision identifiers, used by Alembic. +revision = '2026_07_10_1350' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('directory', + sa.Column('path', wrolpi.media_path.MediaPathType(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('idempotency', wrolpi.dates.TZDateTime(), nullable=True), + sa.PrimaryKeyConstraint('path') + ) + with op.batch_alter_table('directory', schema=None) as batch_op: + batch_op.create_index('directory_name_idx', ['name'], unique=False) + + op.create_table('file_group', + sa.Column('id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), nullable=False), + sa.Column('directory', wrolpi.media_path.MediaPathType(), nullable=False), + sa.Column('author', sa.String(), nullable=True), + sa.Column('censored', sa.Boolean(), nullable=True), + sa.Column('data', wrolpi.files.models.FancyJSON(), nullable=True), + sa.Column('download_datetime', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('files', wrolpi.files.models.FancyJSON(), nullable=True), + sa.Column('idempotency', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('indexed', sa.Boolean(), nullable=True), + sa.Column('length', sa.BigInteger(), nullable=True), + sa.Column('mimetype', sa.String(), nullable=True), + sa.Column('model', sa.String(), nullable=True), + sa.Column('modification_datetime', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('primary_path', wrolpi.media_path.MediaPathType(), nullable=False), + sa.Column('published_datetime', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('published_modified_datetime', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('size', sa.BigInteger(), nullable=True), + sa.Column('suffix', sa.String(), nullable=True), + sa.Column('title', sa.String(), nullable=True), + sa.Column('url', sa.String(), nullable=True), + sa.Column('viewed', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('effective_datetime', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('a_text', sa.String(), nullable=True), + sa.Column('b_text', sa.String(), nullable=True), + sa.Column('c_text', sa.String(), nullable=True), + sa.Column('d_text', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('primary_path') + ) + with op.batch_alter_table('file_group', schema=None) as batch_op: + batch_op.create_index('file_group_author_idx', ['author'], unique=False) + batch_op.create_index('file_group_directory_idx', ['directory'], unique=False) + batch_op.create_index('file_group_download_datetime_idx', ['download_datetime'], unique=False) + batch_op.create_index('file_group_effective_datetime_idx', ['effective_datetime'], unique=False) + batch_op.create_index('file_group_length_idx', ['length'], unique=False) + batch_op.create_index('file_group_mimetype_idx', ['mimetype'], unique=False) + batch_op.create_index('file_group_model_idx', ['model'], unique=False) + batch_op.create_index('file_group_modification_datetime_idx', ['modification_datetime'], unique=False) + batch_op.create_index('file_group_published_datetime_idx', ['published_datetime'], unique=False) + batch_op.create_index('file_group_published_modified_datetime_idx', ['published_modified_datetime'], unique=False) + batch_op.create_index('file_group_size_ix', ['size'], unique=False) + batch_op.create_index('file_group_url_idx', ['url'], unique=False) + batch_op.create_index('file_group_viewed_idx', ['viewed'], unique=False) + batch_op.create_index(batch_op.f('ix_file_group_suffix'), ['suffix'], unique=False) + + op.create_table('tag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('color', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('wrolpi_flag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('refresh_complete', sa.Boolean(), nullable=True), + sa.Column('outdated_zims', sa.Boolean(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('collection', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('kind', sa.String(), server_default='channel', nullable=False), + sa.Column('directory', wrolpi.media_path.MediaPathType(), nullable=True), + sa.Column('tag_id', sa.Integer(), nullable=True), + sa.Column('file_format', sa.String(), nullable=True), + sa.Column('created_date', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), + sa.Column('item_count', sa.Integer(), nullable=False), + sa.Column('total_size', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('directory', name='uq_collection_directory'), + sa.UniqueConstraint('name', 'kind', name='uq_collection_name_kind') + ) + with op.batch_alter_table('collection', schema=None) as batch_op: + batch_op.create_index('idx_collection_item_count', ['item_count'], unique=False) + batch_op.create_index('idx_collection_kind', ['kind'], unique=False) + batch_op.create_index('idx_collection_total_size', ['total_size'], unique=False) + + op.create_table('doc', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('size', sa.BigInteger(), nullable=True), + sa.Column('publisher', sa.String(), nullable=True), + sa.Column('language', sa.String(), nullable=True), + sa.Column('page_count', sa.Integer(), nullable=True), + sa.Column('subject', sa.String(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('file_group_id', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['file_group_id'], ['file_group.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('file_group_id') + ) + with op.batch_alter_table('doc', schema=None) as batch_op: + batch_op.create_index('doc_size_idx', ['size'], unique=False) + + op.create_table('tag_file', + sa.Column('created_at', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('file_group_id', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['file_group_id'], ['file_group.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], ), + sa.PrimaryKeyConstraint('tag_id', 'file_group_id'), + sa.UniqueConstraint('tag_id', 'file_group_id', name='tag_file_tag_id_file_group_id_key') + ) + op.create_table('zim', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('path', wrolpi.media_path.MediaPathType(), nullable=False), + sa.Column('file_group_id', sa.BigInteger(), nullable=False), + sa.Column('auto_search', sa.Boolean(), nullable=True), + sa.ForeignKeyConstraint(['file_group_id'], ['file_group.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('path', name='zim_path_key') + ) + op.create_table('archive', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('collection_id', sa.Integer(), nullable=True), + sa.Column('file_group_id', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['collection_id'], ['collection.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['file_group_id'], ['file_group.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('file_group_id') + ) + with op.batch_alter_table('archive', schema=None) as batch_op: + batch_op.create_index('idx_archive_collection_id', ['collection_id'], unique=False) + + op.create_table('channel', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('url', sa.String(), nullable=True), + sa.Column('generate_posters', sa.Boolean(), nullable=True), + sa.Column('calculate_duration', sa.Boolean(), nullable=True), + sa.Column('download_missing_data', sa.Boolean(), nullable=True), + sa.Column('source_id', sa.String(), nullable=True), + sa.Column('refreshed', sa.Boolean(), nullable=True), + sa.Column('video_count', sa.Integer(), nullable=False), + sa.Column('total_size', sa.BigInteger(), nullable=False), + sa.Column('minimum_frequency', sa.Integer(), nullable=True), + sa.Column('info_json', sa.JSON(), nullable=True), + sa.Column('info_date', sa.Date(), nullable=True), + sa.Column('collection_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['collection_id'], ['collection.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('channel', schema=None) as batch_op: + batch_op.create_index('channel_minimum_frequency_idx', ['minimum_frequency'], unique=False) + batch_op.create_index('channel_source_id', ['source_id'], unique=False) + batch_op.create_index('channel_total_size_idx', ['total_size'], unique=False) + batch_op.create_index('channel_url_key', ['url'], unique=True, postgresql_where=sa.text('url IS NOT NULL'), sqlite_where=sa.text('url IS NOT NULL')) + batch_op.create_index('channel_video_count_idx', ['video_count'], unique=False) + + op.create_table('collection_item', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('collection_id', sa.Integer(), nullable=False), + sa.Column('item_kind', sa.String(), server_default='file', nullable=False), + sa.Column('file_group_id', sa.Integer(), nullable=True), + sa.Column('zim_id', sa.Integer(), nullable=True), + sa.Column('zim_entry', sa.Text(), nullable=True), + sa.Column('url', sa.Text(), nullable=True), + sa.Column('title', sa.Text(), nullable=True), + sa.Column('position', sa.Integer(), nullable=False), + sa.Column('added_date', sa.DateTime(), server_default=sa.text('(CURRENT_TIMESTAMP)'), nullable=False), + sa.CheckConstraint("(item_kind = 'file' AND file_group_id IS NOT NULL AND zim_id IS NULL AND url IS NULL) OR (item_kind = 'zim' AND zim_id IS NOT NULL AND zim_entry IS NOT NULL AND file_group_id IS NULL AND url IS NULL) OR (item_kind = 'url' AND url IS NOT NULL AND file_group_id IS NULL AND zim_id IS NULL)", name='ck_collection_item_kind'), + sa.ForeignKeyConstraint(['collection_id'], ['collection.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['file_group_id'], ['file_group.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['zim_id'], ['zim.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('collection_id', 'file_group_id', name='uq_collection_file_group'), + sa.UniqueConstraint('collection_id', 'url', name='uq_collection_url'), + sa.UniqueConstraint('collection_id', 'zim_id', 'zim_entry', name='uq_collection_zim_entry') + ) + with op.batch_alter_table('collection_item', schema=None) as batch_op: + batch_op.create_index('idx_collection_item_collection_id', ['collection_id'], unique=False) + batch_op.create_index('idx_collection_item_collection_position', ['collection_id', 'position'], unique=False) + batch_op.create_index('idx_collection_item_file_group_id', ['file_group_id'], unique=False) + batch_op.create_index('idx_collection_item_position', ['position'], unique=False) + + op.create_table('doc_section', + sa.Column('id', sa.BigInteger().with_variant(sa.Integer(), 'sqlite'), nullable=False), + sa.Column('doc_id', sa.Integer(), nullable=False), + sa.Column('kind', sa.String(), nullable=False), + sa.Column('ordinal', sa.Integer(), nullable=False), + sa.Column('label', sa.String(), nullable=True), + sa.Column('content', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['doc_id'], ['doc.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('doc_section', schema=None) as batch_op: + batch_op.create_index('doc_section_doc_id_idx', ['doc_id', 'ordinal'], unique=False) + + op.create_table('download', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('url', sa.String(), nullable=False), + sa.Column('attempts', sa.Integer(), nullable=True), + sa.Column('destination', wrolpi.media_path.MediaPathType(), nullable=True), + sa.Column('downloader', sa.Text(), nullable=True), + sa.Column('sub_downloader', sa.Text(), nullable=True), + sa.Column('error', sa.Text(), nullable=True), + sa.Column('frequency', sa.Integer(), nullable=True), + sa.Column('info_json', sa.JSON(), nullable=True), + sa.Column('last_download_attempt', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('last_successful_download', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('location', sa.Text(), nullable=True), + sa.Column('next_download', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('settings', sa.JSON(), nullable=True), + sa.Column('status', sa.String(), nullable=True), + sa.Column('tag_names', sa.JSON(), nullable=True), + sa.Column('collection_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['collection_id'], ['collection.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('url') + ) + with op.batch_alter_table('download', schema=None) as batch_op: + batch_op.create_index('idx_download_collection_id', ['collection_id'], unique=False) + batch_op.create_index('idx_download_last_download_attempt', ['last_download_attempt'], unique=False) + + op.create_table('tag_zim', + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('zim_id', sa.Integer(), nullable=False), + sa.Column('zim_entry', sa.Text(), nullable=False), + sa.Column('created_at', wrolpi.dates.TZDateTime(), nullable=True), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], ), + sa.ForeignKeyConstraint(['zim_id'], ['zim.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('tag_id', 'zim_id', 'zim_entry'), + sa.UniqueConstraint('tag_id', 'zim_id', 'zim_entry', name='tag_zim_tag_id_zim_id_zim_entry_key') + ) + op.create_table('video', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.String(), nullable=True), + sa.Column('view_count', sa.BigInteger(), nullable=True), + sa.Column('ffprobe_json', sa.JSON(), nullable=True), + sa.Column('have_comments', sa.Boolean(), nullable=True), + sa.Column('comments_failed', sa.Boolean(), nullable=True), + sa.Column('channel_id', sa.Integer(), nullable=True), + sa.Column('file_group_id', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['channel_id'], ['channel.id'], ), + sa.ForeignKeyConstraint(['file_group_id'], ['file_group.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('file_group_id'), + sqlite_autoincrement=True + ) + with op.batch_alter_table('video', schema=None) as batch_op: + batch_op.create_index('video_channel_id_idx', ['channel_id'], unique=False) + batch_op.create_index('video_source_id_idx', ['source_id'], unique=False) + batch_op.create_index('video_view_count_idx', ['view_count'], unique=False) + + op.create_table('zim_subscription', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Text(), nullable=False), + sa.Column('language', sa.Text(), nullable=False), + sa.Column('download_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['download_id'], ['download.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + # ### end Alembic commands ### + + # Triggers (channel summaries, effective_datetime) and the FTS5 full-text-search tables. + # Shared with the test databases (wrolpi/conftest.py) so tests exercise the same schema. + from wrolpi.schema_ddl import install_raw_ddl + from wrolpi.fts import rebuild_fts + + connection = op.get_bind() + install_raw_ddl(connection) + # Index any existing content (no-op on a fresh database). + rebuild_fts(connection) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('zim_subscription') + with op.batch_alter_table('video', schema=None) as batch_op: + batch_op.drop_index('video_view_count_idx') + batch_op.drop_index('video_source_id_idx') + batch_op.drop_index('video_channel_id_idx') + + op.drop_table('video') + op.drop_table('tag_zim') + with op.batch_alter_table('download', schema=None) as batch_op: + batch_op.drop_index('idx_download_last_download_attempt') + batch_op.drop_index('idx_download_collection_id') + + op.drop_table('download') + with op.batch_alter_table('doc_section', schema=None) as batch_op: + batch_op.drop_index('doc_section_doc_id_idx') + + op.drop_table('doc_section') + with op.batch_alter_table('collection_item', schema=None) as batch_op: + batch_op.drop_index('idx_collection_item_position') + batch_op.drop_index('idx_collection_item_file_group_id') + batch_op.drop_index('idx_collection_item_collection_position') + batch_op.drop_index('idx_collection_item_collection_id') + + op.drop_table('collection_item') + with op.batch_alter_table('channel', schema=None) as batch_op: + batch_op.drop_index('channel_video_count_idx') + batch_op.drop_index('channel_url_key', postgresql_where=sa.text('url IS NOT NULL'), sqlite_where=sa.text('url IS NOT NULL')) + batch_op.drop_index('channel_total_size_idx') + batch_op.drop_index('channel_source_id') + batch_op.drop_index('channel_minimum_frequency_idx') + + op.drop_table('channel') + with op.batch_alter_table('archive', schema=None) as batch_op: + batch_op.drop_index('idx_archive_collection_id') + + op.drop_table('archive') + op.drop_table('zim') + op.drop_table('tag_file') + with op.batch_alter_table('doc', schema=None) as batch_op: + batch_op.drop_index('doc_size_idx') + + op.drop_table('doc') + with op.batch_alter_table('collection', schema=None) as batch_op: + batch_op.drop_index('idx_collection_total_size') + batch_op.drop_index('idx_collection_kind') + batch_op.drop_index('idx_collection_item_count') + + op.drop_table('collection') + op.drop_table('wrolpi_flag') + op.drop_table('tag') + with op.batch_alter_table('file_group', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_file_group_suffix')) + batch_op.drop_index('file_group_viewed_idx') + batch_op.drop_index('file_group_url_idx') + batch_op.drop_index('file_group_size_ix') + batch_op.drop_index('file_group_published_modified_datetime_idx') + batch_op.drop_index('file_group_published_datetime_idx') + batch_op.drop_index('file_group_modification_datetime_idx') + batch_op.drop_index('file_group_model_idx') + batch_op.drop_index('file_group_mimetype_idx') + batch_op.drop_index('file_group_length_idx') + batch_op.drop_index('file_group_effective_datetime_idx') + batch_op.drop_index('file_group_download_datetime_idx') + batch_op.drop_index('file_group_directory_idx') + batch_op.drop_index('file_group_author_idx') + + op.drop_table('file_group') + with op.batch_alter_table('directory', schema=None) as batch_op: + batch_op.drop_index('directory_name_idx') + + op.drop_table('directory') + # ### end Alembic commands ### diff --git a/alembic/versions/26c5a62ec683_make_filegroup_directory_not_null.py b/alembic/versions/26c5a62ec683_make_filegroup_directory_not_null.py deleted file mode 100644 index 9e61fe98..00000000 --- a/alembic/versions/26c5a62ec683_make_filegroup_directory_not_null.py +++ /dev/null @@ -1,58 +0,0 @@ -"""make_filegroup_directory_not_null - -Revision ID: 26c5a62ec683 -Revises: af4527503bdf -Create Date: 2025-12-23 20:47:11.549264 - -This migration makes the `directory` column NOT NULL after the previous migration -has populated it for all existing FileGroups. -""" -import os - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '26c5a62ec683' -down_revision = 'af4527503bdf' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - # Safety check: ensure no NULL directory values remain - result = session.execute(sa.text( - "SELECT COUNT(*) FROM file_group WHERE directory IS NULL" - )) - null_count = result.scalar() - if null_count > 0: - # Fill any remaining NULL values (shouldn't happen if previous migration ran) - print(f"Warning: Found {null_count} FileGroups with NULL directory, filling from primary_path...") - session.execute(sa.text(""" - UPDATE file_group - SET directory = regexp_replace(primary_path, '/[^/]+$', '') - WHERE directory IS NULL AND primary_path IS NOT NULL - """)) - session.commit() - - # Make the column NOT NULL - op.alter_column('file_group', 'directory', nullable=False) - - # Create index on directory column for efficient directory lookups - op.create_index('file_group_directory_idx', 'file_group', ['directory']) - - if not DOCKERIZED: - session.execute(sa.text('ALTER TABLE public.file_group OWNER TO wrolpi')) - - -def downgrade(): - # Drop the index - op.drop_index('file_group_directory_idx', 'file_group') - # Make the column nullable again - op.alter_column('file_group', 'directory', nullable=True) diff --git a/alembic/versions/2bfb3b50b178_tag_names_migration.py b/alembic/versions/2bfb3b50b178_tag_names_migration.py deleted file mode 100644 index 6bd81b19..00000000 --- a/alembic/versions/2bfb3b50b178_tag_names_migration.py +++ /dev/null @@ -1,137 +0,0 @@ -"""MTag names migration - -Revision ID: 2bfb3b50b178 -Revises: 8d0d81bc9c34 -Create Date: 2024-07-31 14:58:29.583029 - -""" -import os -import pathlib -from datetime import datetime - -import yaml -from alembic import op -from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, String -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Session -from sqlalchemy.orm.exc import NoResultFound # noqa - -from wrolpi import dates -from wrolpi.common import escape_file_name, get_media_directory, ModelHelper -from wrolpi.common import get_relative_to_media_directory -from wrolpi.dates import TZDateTime -from wrolpi.tags import get_tags_config - -# revision identifiers, used by Alembic. -revision = '2bfb3b50b178' -down_revision = '8d0d81bc9c34' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - -Base = declarative_base() - - -class MFileGroup(ModelHelper, Base): - __tablename__ = 'file_group' - id: int = Column(BigInteger, primary_key=True) - primary_path: pathlib.Path = Column(String, unique=True, nullable=False) - - -class MTagFile(ModelHelper, Base): - __tablename__ = 'tag_file' - tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True) - file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), primary_key=True) - - -class MTag(ModelHelper, Base): - __tablename__ = 'tag' - id = Column(Integer, primary_key=True) - name = Column(String, unique=True, nullable=False) - color = Column(String) - - -class MZim(Base): - __tablename__ = 'zim' - id = Column(Integer, primary_key=True) - path: pathlib.Path = Column(String, unique=True, nullable=False) - - -class MTagZimEntry(Base): - __tablename__ = 'tag_zim' - tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True) - zim_id = Column(Integer, ForeignKey('zim.id', ondelete='CASCADE'), primary_key=True) - zim_entry: str = Column(Text, nullable=False, primary_key=True) - created_at: datetime = Column(TZDateTime, default=dates.now) - - -def save_tags(session: Session): - media_directory = get_media_directory() - - tags = dict() - tag_rows = session.query(MTag) - for tag in tag_rows: - tags[tag.name] = dict(color=tag.color) - - results = session.query(MTag, MTagFile, MFileGroup) \ - .filter(MTagFile.tag_id == MTag.id, MTagFile.file_group_id == MFileGroup.id) \ - .order_by(MFileGroup.primary_path) - - tag_files = [] - for tag, tag_file, file_group in results: - value = [ - tag.name, - str(file_group.primary_path.relative_to(media_directory)), - # Fallback to current time if not set. - tag_file.created_at.isoformat() if tag_file.created_at else dates.now().isoformat(), - ] - tag_files.append(value) - - results = session.query(MTag, MZim, MTagZimEntry) \ - .filter(MTag.id == MTagZimEntry.tag_id, MZim.id == MTagZimEntry.zim_id) \ - .order_by(MTagZimEntry.zim_id, MTagZimEntry.zim_entry) - - tag_zims = [] - for tag, zim, tag_zim_entry in results: - zim: MZim - tag_zim_entry: MTagZimEntry - value = [ - tag.name, - str(get_relative_to_media_directory(zim.path)), - tag_zim_entry.zim_entry, - # Fallback to current time if not set. - tag_zim_entry.created_at.isoformat() if tag_zim_entry.created_at else dates.now().isoformat(), - ] - tag_zims.append(value) - - # Write to the config. - with get_tags_config().get_file().open('wt') as fh: - data = { - 'tag_files': tag_files, - 'tag_zims': tag_zims, - 'tags': tags, - } - yaml.dump(data, fh) - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - need_commit = False - tags = session.query(MTag).all() - for tag in tags: - escaped_name = escape_file_name(tag.name) - if escaped_name != tag.name: - tag.name = escaped_name - session.flush([tag, ]) - need_commit = True - - if need_commit: - session.commit() - save_tags(session) - - -def downgrade(): - pass diff --git a/alembic/versions/2fb39c1861c7_create_mapfile_table.py b/alembic/versions/2fb39c1861c7_create_mapfile_table.py deleted file mode 100644 index 3f362f0c..00000000 --- a/alembic/versions/2fb39c1861c7_create_mapfile_table.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Create MapFile table. - -Revision ID: 2fb39c1861c7 -Revises: 36cae041ad31 -Create Date: 2022-03-03 19:13:39.997580 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '2fb39c1861c7' -down_revision = '36cae041ad31' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('''CREATE TABLE map_file ( - id SERIAL PRIMARY KEY, - path TEXT NOT NULL, - imported BOOLEAN DEFAULT FALSE, - size BIGINT - )''') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.map_file OWNER TO wrolpi') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('DROP TABLE IF EXISTS map_file') diff --git a/alembic/versions/31896cc29b71_archive_contents_search.py b/alembic/versions/31896cc29b71_archive_contents_search.py deleted file mode 100644 index 89fe8c50..00000000 --- a/alembic/versions/31896cc29b71_archive_contents_search.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Archive contents search. - -Revision ID: 31896cc29b71 -Revises: bddb5e95a91e -Create Date: 2021-12-31 18:53:25.326674 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '31896cc29b71' -down_revision = 'bddb5e95a91e' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE archive ADD COLUMN contents TEXT') - session.execute(''' - ALTER TABLE archive - ADD COLUMN textsearch tsvector - GENERATED ALWAYS AS ( - setweight(to_tsvector('english'::regconfig, title), 'A') || - setweight(to_tsvector('english'::regconfig, contents), 'D') - ) STORED - ''') - session.execute('CREATE INDEX IF NOT EXISTS archive_textsearch_idx ON archive USING GIN(textsearch)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('DROP INDEX IF EXISTS archive_textsearch_idx') - session.execute('ALTER TABLE archive DROP COLUMN IF EXISTS contents') - session.execute('ALTER TABLE archive DROP COLUMN IF EXISTS textsearch') diff --git a/alembic/versions/32d96eb9122d_video_censored.py b/alembic/versions/32d96eb9122d_video_censored.py deleted file mode 100644 index b341df48..00000000 --- a/alembic/versions/32d96eb9122d_video_censored.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Video censored - -Revision ID: 32d96eb9122d -Revises: 184329ed5137 -Create Date: 2021-12-23 14:03:53.677547 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '32d96eb9122d' -down_revision = '184329ed5137' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE video ADD COLUMN censored BOOL DEFAULT false') - session.execute('CREATE INDEX video_censored ON video (censored)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS censored') - session.execute('DROP INDEX IF EXISTS video_censored') diff --git a/alembic/versions/365c985893b8_ebook_creator.py b/alembic/versions/365c985893b8_ebook_creator.py deleted file mode 100644 index 00a9b48d..00000000 --- a/alembic/versions/365c985893b8_ebook_creator.py +++ /dev/null @@ -1,49 +0,0 @@ -"""eBook creator. - -Revision ID: 365c985893b8 -Revises: 5ad9efbf3570 -Create Date: 2023-10-29 16:54:28.319427 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '365c985893b8' -down_revision = '5ad9efbf3570' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - UPDATE file_group - SET author=creator - FROM ebook - WHERE ebook.file_group_id = file_group.id - ''') - session.execute('DROP INDEX IF EXISTS ebook_creator_idx') - session.execute('ALTER TABLE ebook DROP COLUMN IF EXISTS creator') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE ebook ADD COLUMN IF NOT EXISTS creator TEXT') - session.execute(''' - UPDATE ebook - SET creator=author - FROM file_group - WHERE - ebook.file_group_id = file_group.id - AND model = 'ebook' - ''') - session.execute('CREATE INDEX IF NOT EXISTS ebook_creator_idx ON ebook(creator)') diff --git a/alembic/versions/36cae041ad31_files.py b/alembic/versions/36cae041ad31_files.py deleted file mode 100644 index 8e69db38..00000000 --- a/alembic/versions/36cae041ad31_files.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Files - -Revision ID: 36cae041ad31 -Revises: b97c1313f986 -Create Date: 2022-02-01 11:05:57.605541 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '36cae041ad31' -down_revision = 'b97c1313f986' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('''CREATE TABLE file ( - id BIGSERIAL PRIMARY KEY, - path TEXT NOT NULL, - title TEXT, - mimetype TEXT, - size INTEGER, - modification_datetime TIMESTAMPTZ, - textsearch tsvector GENERATED ALWAYS AS ( - setweight(to_tsvector('english'::regconfig, title), 'A'::"char") - ) stored, - idempotency TEXT - )''') - session.execute('CREATE INDEX file_path_idx ON file(path)') - session.execute('CREATE INDEX file_mimetype_idx ON file(textsearch)') - session.execute('CREATE INDEX file_textsearch_idx ON file(textsearch)') - if not DOCKERIZED: - session.execute('ALTER TABLE public.file OWNER TO wrolpi') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP INDEX IF EXISTS file_textsearch_idx') - session.execute('DROP INDEX IF EXISTS file_mimetype_idx') - session.execute('DROP INDEX IF EXISTS file_path_idx') - - session.execute('DROP TABLE IF EXISTS file') diff --git a/alembic/versions/3b6918aeca4b_file_group.py b/alembic/versions/3b6918aeca4b_file_group.py deleted file mode 100644 index 2364325f..00000000 --- a/alembic/versions/3b6918aeca4b_file_group.py +++ /dev/null @@ -1,242 +0,0 @@ -"""File group. - -Revision ID: 3b6918aeca4b -Revises: 930b9b7adb79 -Create Date: 2023-02-27 14:52:46.943351 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '3b6918aeca4b' -down_revision = '930b9b7adb79' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DELETE FROM video') # noqa - session.execute('ALTER TABLE video DROP COLUMN caption_path') - session.execute('ALTER TABLE video DROP COLUMN censored') - session.execute('ALTER TABLE video DROP COLUMN favorite') - session.execute('ALTER TABLE video DROP COLUMN info_json_path') - session.execute('ALTER TABLE video DROP COLUMN modification_datetime') - session.execute('ALTER TABLE video DROP COLUMN poster_path') - session.execute('ALTER TABLE video DROP COLUMN size') - session.execute('ALTER TABLE video DROP COLUMN title') - session.execute('ALTER TABLE video DROP COLUMN validated') - session.execute('ALTER TABLE video DROP COLUMN video_path') - - session.execute('DELETE FROM archive') # noqa - session.execute('ALTER TABLE archive DROP COLUMN readability_json_path') - session.execute('ALTER TABLE archive DROP COLUMN readability_path') - session.execute('ALTER TABLE archive DROP COLUMN readability_txt_path') - session.execute('ALTER TABLE archive DROP COLUMN screenshot_path') - session.execute('ALTER TABLE archive DROP COLUMN singlefile_path') - session.execute('ALTER TABLE archive DROP COLUMN title') - session.execute('ALTER TABLE archive DROP COLUMN validated') - - session.execute('DELETE FROM ebook') # noqa - session.execute('ALTER TABLE ebook DROP COLUMN cover_path') - session.execute('ALTER TABLE ebook DROP COLUMN ebook_path') - session.execute('ALTER TABLE ebook DROP COLUMN title') - - session.execute('DROP TABLE IF EXISTS file') - - session.execute(''' - CREATE TABLE file_group ( - id BIGSERIAL PRIMARY KEY, - data JSON, - files JSON, - idempotency TIMESTAMP WITH TIME ZONE, - indexed BOOLEAN DEFAULT FALSE, - mimetype TEXT NOT NULL, - model TEXT, - modification_datetime TIMESTAMP WITH TIME ZONE, - primary_path TEXT NOT NULL UNIQUE, - size BIGINT, - title TEXT, - a_text TEXT, - b_text TEXT, - c_text TEXT, - d_text TEXT, - textsearch tsvector GENERATED ALWAYS AS ( - setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char") - ) STORED - )''') - - session.execute( - 'ALTER TABLE video ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE') - session.execute( - 'ALTER TABLE ebook ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE') - session.execute( - 'ALTER TABLE archive ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE') - - session.execute(''' - CREATE TABLE tag ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL UNIQUE, - color VARCHAR(7) - )''') - - session.execute(''' - CREATE TABLE tag_file ( - tag_id INTEGER REFERENCES tag(id), - file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE, - created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - UNIQUE (tag_id, file_group_id) - )''') - - session.execute(''' - INSERT INTO tag (name, color) VALUES - ('Automotive', '#353535'), - ('Clothing', '#9858C5'), - ('Comms', '#ECD71C'), - ('Cooking', '#00A04A'), - ('Electronics', '#1b731f'), - ('Entertainment', '#981473'), - ('Favorite', '#000000'), - ('Fire Safety', '#d16711'), - ('Firearms', '#6c6c6c'), - ('First Aid', '#C40405'), - ('Food Preservation', '#676816'), - ('Food', '#2F8055'), - ('Fuel', '#c47fb3'), - ('Garden', '#3E8B22'), - ('Heating', '#ff5400'), - ('Husbandry','#B58039'), - ('Hygiene', '#adadad'), - ('Knots', '#9d6a4c'), - ('Light', '#EEEEEE'), - ('Mechanical', '#7A7A7A'), - ('Medical', '#4a869c'), - ('Navigation', '#1ab3ce'), - ('Nuclear', '#0dff00'), - ('Pests', '#050068'), - ('Power', '#D837FF'), - ('Primitive', '#A18A62'), - ('Radio', '#134cbb'), - ('Schooling', '#6d752e'), - ('Security', '#333333'), - ('Shelter', '#625641'), - ('Social', '#490c73'), - ('Tools', '#f0d730'), - ('WROL', '#4700ab'), - ('Water', '#291BC2') - ''') - - session.execute('UPDATE wrolpi_flag SET refresh_complete = false') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.file_group OWNER TO wrolpi') - session.execute('ALTER TABLE public.tag OWNER TO wrolpi') - session.execute('ALTER TABLE public.tag_file OWNER TO wrolpi') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP TABLE IF EXISTS tag_file') - session.execute('DROP TABLE IF EXISTS tag') - - session.execute('ALTER TABLE video DROP COLUMN file_group_id') - session.execute('ALTER TABLE ebook DROP COLUMN file_group_id') - session.execute('ALTER TABLE archive DROP COLUMN file_group_id') - - session.execute('DROP TABLE file_group') - - session.execute(''' - create table file - ( - path text not null primary key, - idempotency timestamp with time zone, - indexed boolean default false, - mimetype text, - suffix text, - model text, - associated boolean default false, - modification_datetime timestamp with time zone, - size bigint, - title text, - a_text text, - b_text text, - c_text text, - d_text text, - textsearch tsvector generated always as (( - ((setweight(to_tsvector('english'::regconfig, COALESCE(a_text, ''::text)), 'A'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(b_text, ''::text)), 'B'::"char")) || - setweight(to_tsvector('english'::regconfig, COALESCE(c_text, ''::text)), 'C'::"char")) || - setweight(to_tsvector('english'::regconfig, COALESCE(d_text, ''::text)), 'D'::"char"))) stored, - full_stem text - ); - - alter table file - owner to postgres; - - create index file_mimetype_idx - on file (mimetype); - - create index file_textsearch_idx - on file using gin (textsearch); - - create index file_modification_datetime_idx - on file (modification_datetime); - - create index file_size_idx - on file (size); - - create index file_model_idx - on file (model); - - create index file_idempotency_idx - on file (idempotency); - - create index file_suffix_idx - on file (suffix); - - create index file_full_stem_idx - on file (full_stem); - ''') - - session.execute('DELETE FROM video') # noqa - session.execute('ALTER TABLE video ADD COLUMN caption_path TEXT REFERENCES file(path)') - session.execute('ALTER TABLE video ADD COLUMN censored BOOLEAN DEFAULT FALSE') - session.execute('ALTER TABLE video ADD COLUMN favorite TIMESTAMPTZ') - session.execute('ALTER TABLE video ADD COLUMN info_json_path TEXT REFERENCES file(path)') - session.execute('ALTER TABLE video ADD COLUMN poster_path TEXT REFERENCES file(path)') - session.execute('ALTER TABLE video ADD COLUMN video_path TEXT REFERENCES file(path)') - session.execute('ALTER TABLE video ADD COLUMN modification_datetime TIMESTAMPTZ') - session.execute('ALTER TABLE video ADD COLUMN size BIGINT') - session.execute('ALTER TABLE video ADD COLUMN title TEXT') - session.execute('ALTER TABLE video ADD COLUMN validated BOOLEAN DEFAULT FALSE') - - session.execute('DELETE FROM archive') # noqa - session.execute('ALTER TABLE archive ADD COLUMN readability_json_path TEXT REFERENCES file(path)') - session.execute('ALTER TABLE archive ADD COLUMN readability_path TEXT REFERENCES file(path)') - session.execute('ALTER TABLE archive ADD COLUMN readability_txt_path TEXT REFERENCES file(path)') - session.execute('ALTER TABLE archive ADD COLUMN screenshot_path TEXT REFERENCES file(path)') - session.execute('ALTER TABLE archive ADD COLUMN singlefile_path TEXT REFERENCES file(path)') - session.execute('ALTER TABLE archive ADD COLUMN title TEXT') - session.execute('ALTER TABLE archive ADD COLUMN validated BOOLEAN DEFAULT FALSE') - - session.execute('DELETE FROM ebook') # noqa - session.execute('ALTER TABLE ebook ADD COLUMN cover_path TEXT REFERENCES file(path)') - session.execute('ALTER TABLE ebook ADD COLUMN ebook_path TEXT REFERENCES file(path)') - session.execute('ALTER TABLE ebook ADD COLUMN title TEXT') - - session.execute('UPDATE wrolpi_flag SET refresh_complete = false') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.file OWNER TO wrolpi') diff --git a/alembic/versions/3c33b1dae069_video_validated.py b/alembic/versions/3c33b1dae069_video_validated.py deleted file mode 100644 index 5f6aa7ae..00000000 --- a/alembic/versions/3c33b1dae069_video_validated.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Video validated. - -Revision ID: 3c33b1dae069 -Revises: f2c6326cad29 -Create Date: 2022-01-13 16:14:23.409242 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '3c33b1dae069' -down_revision = 'f2c6326cad29' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE video ADD COLUMN validated BOOLEAN DEFAULT FALSE') - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS validated_poster') - # Generating posters is slow, and may delete files. - session.execute('ALTER TABLE channel ALTER COLUMN generate_posters SET DEFAULT false') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS validated') - session.execute('ALTER TABLE video ADD COLUMN validated_poster BOOLEAN DEFAULT FALSE') diff --git a/alembic/versions/467235201c70_download_sub_downloader.py b/alembic/versions/467235201c70_download_sub_downloader.py deleted file mode 100644 index c8f11c34..00000000 --- a/alembic/versions/467235201c70_download_sub_downloader.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Download.sub_downloader. - -Revision ID: 467235201c70 -Revises: 1a70196c05bb -Create Date: 2022-05-20 11:51:58.064020 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '467235201c70' -down_revision = '1a70196c05bb' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - session = Session(bind=op.get_bind()) - session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS sub_downloader TEXT') - - -def downgrade(): - session = Session(bind=op.get_bind()) - session.execute('ALTER TABLE download DROP COLUMN IF EXISTS sub_downloader') diff --git a/alembic/versions/4b54ad052e2e_reindex_archives.py b/alembic/versions/4b54ad052e2e_reindex_archives.py deleted file mode 100644 index e84d808c..00000000 --- a/alembic/versions/4b54ad052e2e_reindex_archives.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Reindex archives. - -Revision ID: 4b54ad052e2e -Revises: 6b0b249a4f81 -Create Date: 2023-10-20 13:28:55.517086 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '4b54ad052e2e' -down_revision = '6b0b249a4f81' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - # Archives must be re-indexed because of the previous revision. - session.execute("UPDATE file_group SET indexed=false WHERE model='archive'") - session.execute("DELETE FROM archive") # noqa - - -def downgrade(): - pass diff --git a/alembic/versions/4c8da5502c8a_drop_channel_skip_download_videos.py b/alembic/versions/4c8da5502c8a_drop_channel_skip_download_videos.py deleted file mode 100644 index 4e4105ee..00000000 --- a/alembic/versions/4c8da5502c8a_drop_channel_skip_download_videos.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Drop Channel.skip_download_videos. - -Revision ID: 4c8da5502c8a -Revises: 8bdb48a486b9 -Create Date: 2023-09-29 14:45:42.999526 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '4c8da5502c8a' -down_revision = '8bdb48a486b9' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS skip_download_videos') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS skip_download_videos TEXT[]') diff --git a/alembic/versions/4f03b9548f6e_file_group_effective_date_trigger.py b/alembic/versions/4f03b9548f6e_file_group_effective_date_trigger.py deleted file mode 100644 index 61945eaa..00000000 --- a/alembic/versions/4f03b9548f6e_file_group_effective_date_trigger.py +++ /dev/null @@ -1,47 +0,0 @@ -"""file_group.effective_date trigger - -Revision ID: 4f03b9548f6e -Revises: 54c19aa847da -Create Date: 2025-08-17 10:49:35.754548 - -""" -import os -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '4f03b9548f6e' -down_revision = '54c19aa847da' -branch_labels = None -depends_on = None - - -def upgrade(): - # Create trigger function - op.execute(""" - CREATE OR REPLACE FUNCTION update_effective_datetime() - RETURNS TRIGGER AS - $$ - BEGIN - NEW.effective_datetime = COALESCE(NEW.published_datetime, NEW.download_datetime); - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """) - - # Create trigger - op.execute(""" - CREATE TRIGGER file_group_effective_datetime_trigger - BEFORE INSERT OR UPDATE OF published_datetime, download_datetime - ON file_group - FOR EACH ROW - EXECUTE FUNCTION update_effective_datetime(); - """) - - -def downgrade(): - # Drop trigger - op.execute("DROP TRIGGER IF EXISTS file_group_effective_datetime_trigger ON file_group") - - # Drop trigger function - op.execute("DROP FUNCTION IF EXISTS update_effective_datetime") diff --git a/alembic/versions/53f6780aa599_drop_urls.py b/alembic/versions/53f6780aa599_drop_urls.py deleted file mode 100644 index f5b33ce7..00000000 --- a/alembic/versions/53f6780aa599_drop_urls.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Drop URLs. - -Revision ID: 53f6780aa599 -Revises: 31896cc29b71 -Create Date: 2022-01-03 12:25:51.575505 - -""" -import os - -from alembic import op -from sqlalchemy import text -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '53f6780aa599' -down_revision = '31896cc29b71' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - urls = {i['id']: i['url'] for i in session.execute(text('SELECT id, url FROM url'))} - archives = {i['id']: i['url_id'] for i in session.execute(text('SELECT id, url_id FROM archive'))} - - session.execute(text('ALTER TABLE archive ADD COLUMN url TEXT')) - - for archive_id, url_id in archives.items(): - session.execute( - text('UPDATE archive SET url = :url WHERE id = :id'), - {'url': urls[url_id], 'id': archive_id} - ) - - session.execute(text('ALTER TABLE archive DROP COLUMN url_id')) - session.execute(text('DROP TABLE url')) diff --git a/alembic/versions/54c19aa847da_channel_summary_indexes.py b/alembic/versions/54c19aa847da_channel_summary_indexes.py deleted file mode 100644 index dc00e745..00000000 --- a/alembic/versions/54c19aa847da_channel_summary_indexes.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Channel summary columns, triggers and indexes. - -Revision ID: 54c19aa847da -Revises: 158973fda20f -Create Date: 2025-08-16 11:45:48.190427 - -""" -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision = '54c19aa847da' -down_revision = '158973fda20f' -branch_labels = None -depends_on = None - - -def upgrade(): - # Add columns - op.execute(""" - ALTER TABLE channel - ADD COLUMN video_count INTEGER DEFAULT 0 NOT NULL, - ADD COLUMN total_size BIGINT DEFAULT 0 NOT NULL, - ADD COLUMN minimum_frequency INTEGER; - """) - - # Populate columns - op.execute(""" - UPDATE channel - SET video_count = (SELECT COUNT(*) - FROM video - WHERE channel_id = channel.id), - total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT - FROM video v - LEFT JOIN file_group fg ON v.file_group_id = fg.id - WHERE v.channel_id = channel.id), - minimum_frequency = (SELECT MIN(frequency) - FROM download - WHERE channel_id = channel.id); - """) - - # Create trigger functions - op.execute(""" - CREATE OR REPLACE FUNCTION update_channel_video_count() - RETURNS TRIGGER AS - $$ - BEGIN - UPDATE channel - SET video_count = (SELECT COUNT(*) FROM video WHERE channel_id = NEW.channel_id) - WHERE id = NEW.channel_id; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """) - op.execute(""" - CREATE OR REPLACE FUNCTION update_channel_size() - RETURNS TRIGGER AS - $$ - BEGIN - UPDATE channel - SET total_size = (SELECT COALESCE(SUM(fg.size), 0)::BIGINT - FROM video v - LEFT JOIN file_group fg ON v.file_group_id = fg.id - WHERE v.channel_id = NEW.id) - WHERE id = NEW.id; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """) - op.execute(""" - CREATE OR REPLACE FUNCTION update_channel_minimum_frequency() - RETURNS TRIGGER AS - $$ - BEGIN - UPDATE channel - SET minimum_frequency = (SELECT MIN(frequency) FROM download WHERE channel_id = NEW.channel_id) - WHERE id = NEW.channel_id; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """) - - # Create triggers - op.execute(""" - CREATE TRIGGER video_count_trigger - AFTER INSERT OR DELETE - ON video - FOR EACH ROW - EXECUTE FUNCTION update_channel_video_count(); - """) - op.execute(""" - CREATE TRIGGER video_size_trigger - AFTER INSERT OR DELETE - ON video - FOR EACH ROW - EXECUTE FUNCTION update_channel_size(); - """) - op.execute(""" - CREATE TRIGGER file_group_size_trigger - AFTER UPDATE OF size - ON file_group - FOR EACH ROW - WHEN (OLD.size IS DISTINCT FROM NEW.size) - EXECUTE FUNCTION update_channel_size(); - """) - op.execute(""" - CREATE TRIGGER download_frequency_trigger - AFTER INSERT OR UPDATE OF frequency OR DELETE - ON download - FOR EACH ROW - EXECUTE FUNCTION update_channel_minimum_frequency(); - """) - - # Create indexes - op.execute(""" - CREATE INDEX channel_video_count_idx ON channel (video_count); - CREATE INDEX channel_total_size_idx ON channel (total_size); - CREATE INDEX channel_minimum_frequency_idx ON channel (minimum_frequency); - """) - - -def downgrade(): - # Drop triggers - op.execute("DROP TRIGGER IF EXISTS video_count_trigger ON video;") - op.execute("DROP TRIGGER IF EXISTS video_size_trigger ON video;") - op.execute("DROP TRIGGER IF EXISTS file_group_size_trigger ON file_group;") - op.execute("DROP TRIGGER IF EXISTS download_frequency_trigger ON download;") - - # Drop trigger functions - op.execute("DROP FUNCTION IF EXISTS update_channel_video_count;") - op.execute("DROP FUNCTION IF EXISTS update_channel_size;") - op.execute("DROP FUNCTION IF EXISTS update_channel_minimum_frequency;") - - # Drop indexes - op.execute("DROP INDEX IF EXISTS channel_minimum_frequency_idx;") - op.execute("DROP INDEX IF EXISTS channel_total_size_idx;") - op.execute("DROP INDEX IF EXISTS channel_video_count_idx;") - - # Drop columns - op.execute(""" - ALTER TABLE channel - DROP COLUMN IF EXISTS minimum_frequency, - DROP COLUMN IF EXISTS total_size, - DROP COLUMN IF EXISTS video_count; - """) diff --git a/alembic/versions/577e414f1bc4_add_archive_collection_id_index.py b/alembic/versions/577e414f1bc4_add_archive_collection_id_index.py deleted file mode 100644 index e9b3081c..00000000 --- a/alembic/versions/577e414f1bc4_add_archive_collection_id_index.py +++ /dev/null @@ -1,26 +0,0 @@ -"""add_archive_collection_id_index - -Revision ID: 577e414f1bc4 -Revises: f38cfe4b5cb2 -Create Date: 2025-12-07 17:25:27.809843 - -""" -from alembic import op - - -# revision identifiers, used by Alembic. -revision = '577e414f1bc4' -down_revision = 'f38cfe4b5cb2' -branch_labels = None -depends_on = None - - -def upgrade(): - # Add index on archive.collection_id to speed up GROUP BY queries - # and per-collection lookups. This index was missing and causing - # full table scans on the archive table (115k+ rows). - op.create_index('idx_archive_collection_id', 'archive', ['collection_id']) - - -def downgrade(): - op.drop_index('idx_archive_collection_id', table_name='archive') diff --git a/alembic/versions/5ad9efbf3570_filegroup_length.py b/alembic/versions/5ad9efbf3570_filegroup_length.py deleted file mode 100644 index 8381fdca..00000000 --- a/alembic/versions/5ad9efbf3570_filegroup_length.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -Migrates Video.duration/.url/.viewed/Archive.url to FileGroup.length/.url/.viewed. - -Revision ID: 5ad9efbf3570 -Revises: 4b54ad052e2e -Create Date: 2023-10-24 09:25:33.818373 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '5ad9efbf3570' -down_revision = '4b54ad052e2e' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE file_group ADD COLUMN length INT DEFAULT 0') - session.execute(''' - UPDATE file_group - SET length=video.duration - FROM video - WHERE file_group.id = video.file_group_id - ''') - - # Migrate Archive.url and Video.url to FileGroup.url. - session.execute('ALTER TABLE file_group ADD COLUMN url TEXT') - session.execute(''' - UPDATE file_group - SET url=archive.url - FROM archive - WHERE file_group.id = archive.file_group_id - ''') - session.execute(''' - UPDATE file_group - SET url=video.url - FROM video - WHERE file_group.id = video.file_group_id - ''') - - # Migrate Archive.archive_datetime to FileGroup.download_datetime. - session.execute('ALTER TABLE file_group ADD COLUMN download_datetime TIMESTAMP WITH TIME ZONE') - session.execute(''' - UPDATE file_group - SET download_datetime=archive.archive_datetime - FROM archive - WHERE file_group.id = archive.file_group_id - ''') - - # Migrate Video.viewed to FileGroup.viewed. - session.execute('ALTER TABLE file_group ADD COLUMN viewed TIMESTAMP WITH TIME ZONE') - session.execute(''' - UPDATE file_group - SET viewed=video.viewed - FROM video - WHERE file_group.id = video.file_group_id - ''') - - session.execute('DROP INDEX IF EXISTS archive_archive_datetime_idx') - session.execute('DROP INDEX IF EXISTS archive_url_idx') - session.execute('DROP INDEX IF EXISTS video_url_idx') - session.execute('DROP INDEX IF EXISTS video_viewed_idx') - session.execute('ALTER TABLE archive DROP COLUMN IF EXISTS archive_datetime') - session.execute('ALTER TABLE archive DROP COLUMN IF EXISTS url') - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS url') - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS viewed') - - session.execute('CREATE INDEX file_group_download_datetime_idx ON file_group(download_datetime)') - session.execute('CREATE INDEX file_group_length_idx ON file_group(length)') - session.execute('CREATE INDEX file_group_url_idx ON file_group(url)') - session.execute('CREATE INDEX file_group_viewed_idx ON file_group(viewed)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE video ADD COLUMN IF NOT EXISTS duration INTEGER') - session.execute(''' - UPDATE video - SET duration=file_group.length - FROM file_group - WHERE file_group.id = video.file_group_id - ''') - - session.execute('ALTER TABLE video ADD COLUMN IF NOT EXISTS url TEXT') - session.execute(''' - UPDATE video - SET url=file_group.url - FROM file_group - WHERE file_group.id = video.file_group_id - ''') - - # Video has `viewed`, Archive does not. - session.execute('ALTER TABLE video ADD COLUMN IF NOT EXISTS viewed TIMESTAMP WITH TIME ZONE') - session.execute(''' - UPDATE video - SET url=file_group.url - FROM file_group - WHERE file_group.id = video.file_group_id - ''') - - session.execute('ALTER TABLE archive ADD COLUMN IF NOT EXISTS url TEXT') - session.execute(''' - UPDATE archive - SET url=file_group.url - FROM file_group - WHERE file_group.id = archive.file_group_id - ''') - - session.execute('ALTER TABLE archive ADD COLUMN IF NOT EXISTS archive_datetime TIMESTAMP WITH TIME ZONE') - session.execute(''' - UPDATE archive - SET archive_datetime=file_group.download_datetime - FROM file_group - WHERE file_group.id = archive.file_group_id - ''') - - session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS download_datetime') - session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS length') - session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS url') - session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS viewed') - session.execute('DROP INDEX IF EXISTS file_group_download_datetime_idx') - session.execute('DROP INDEX IF EXISTS file_group_length_idx') - session.execute('DROP INDEX IF EXISTS file_group_url_idx') - session.execute('DROP INDEX IF EXISTS file_group_viewed_idx') - - session.execute('CREATE INDEX IF NOT EXISTS archive_archive_datetime_idx ON archive(archive_datetime)') - session.execute('CREATE INDEX IF NOT EXISTS archive_url_idx ON archive(url)') - session.execute('CREATE INDEX IF NOT EXISTS video_duration_idx ON video(duration)') - session.execute('CREATE INDEX IF NOT EXISTS video_url_idx ON video(url)') - session.execute('CREATE INDEX IF NOT EXISTS video_viewed_idx ON video(viewed)') diff --git a/alembic/versions/62acddc55091_archive_textsearch_coalesce_fix.py b/alembic/versions/62acddc55091_archive_textsearch_coalesce_fix.py deleted file mode 100644 index 61422bc0..00000000 --- a/alembic/versions/62acddc55091_archive_textsearch_coalesce_fix.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Archive textsearch coalesce fix. - -Revision ID: 62acddc55091 -Revises: 0400005fe2dc -Create Date: 2022-01-04 11:11:29.100317 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '62acddc55091' -down_revision = '0400005fe2dc' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('DROP INDEX IF EXISTS archive_textsearch_idx') - session.execute('ALTER TABLE archive DROP COLUMN IF EXISTS textsearch') - session.execute(''' - ALTER TABLE archive - ADD COLUMN textsearch tsvector - GENERATED ALWAYS AS ( - setweight(to_tsvector('english'::regconfig, COALESCE(title, '')), 'A') || - setweight(to_tsvector('english'::regconfig, COALESCE(contents, '')), 'D') - ) STORED - ''') - session.execute('CREATE INDEX archive_textsearch_idx ON archive USING GIN(textsearch)') - - -def downgrade(): - # The upgrade is backwards compatible. - pass diff --git a/alembic/versions/6489cd50c889_file_stem_and_directory.py b/alembic/versions/6489cd50c889_file_stem_and_directory.py deleted file mode 100644 index 20dc7db1..00000000 --- a/alembic/versions/6489cd50c889_file_stem_and_directory.py +++ /dev/null @@ -1,43 +0,0 @@ -"""File stem and directory - -Revision ID: 6489cd50c889 -Revises: 13542a230c78 -Create Date: 2022-12-04 15:39:55.485355 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '6489cd50c889' -down_revision = '13542a230c78' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE file ADD COLUMN directory TEXT') - session.execute('ALTER TABLE file ADD COLUMN full_stem TEXT') - session.execute('ALTER TABLE file ADD COLUMN stem TEXT') - session.execute('CREATE INDEX file_directory_idx ON file(directory)') - session.execute('CREATE INDEX file_full_stem_idx ON file(full_stem)') - session.execute('CREATE INDEX file_stem_idx ON file(stem)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP INDEX IF EXISTS file_directory_idx') - session.execute('DROP INDEX IF EXISTS file_full_stem_idx') - session.execute('DROP INDEX IF EXISTS file_stem_idx') - session.execute('ALTER TABLE file DROP COLUMN IF EXISTS directory') - session.execute('ALTER TABLE file DROP COLUMN IF EXISTS full_stem') - session.execute('ALTER TABLE file DROP COLUMN IF EXISTS stem') diff --git a/alembic/versions/66407d145b76_.py b/alembic/versions/66407d145b76_.py deleted file mode 100644 index bd2ea5b3..00000000 --- a/alembic/versions/66407d145b76_.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Create collection table for domain collections - -Revision ID: 66407d145b76 -Revises: 4f03b9548f6e -Create Date: 2025-10-26 10:57:16.462524 - -""" -import os -from alembic import op -import sqlalchemy as sa -from sqlalchemy.orm import Session - - -# revision identifiers, used by Alembic. -revision = '66407d145b76' -down_revision = '4f03b9548f6e' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - # Step 1: Create new collection table (keeping channel table separate for now) - op.create_table( - 'collection', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(), nullable=False), - sa.Column('description', sa.Text(), nullable=True), - sa.Column('kind', sa.String(), nullable=False), - sa.Column('directory', sa.Text(), nullable=True), - sa.Column('tag_id', sa.Integer(), nullable=True), - sa.Column('created_date', sa.DateTime(), server_default=sa.func.now(), nullable=False), - sa.Column('item_count', sa.Integer(), nullable=False, server_default='0'), - sa.Column('total_size', sa.Integer(), nullable=False, server_default='0'), - sa.PrimaryKeyConstraint('id'), - sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name='collection_tag_id_fkey'), - sa.UniqueConstraint('directory', name='uq_collection_directory') - ) - - # Create indexes for collection - op.create_index('idx_collection_kind', 'collection', ['kind'], unique=False) - op.create_index('idx_collection_item_count', 'collection', ['item_count'], unique=False) - op.create_index('idx_collection_total_size', 'collection', ['total_size'], unique=False) - - # Step 2: Create collection_item junction table - op.create_table( - 'collection_item', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('collection_id', sa.Integer(), nullable=False), - sa.Column('file_group_id', sa.Integer(), nullable=False), - sa.Column('position', sa.Integer(), nullable=False, server_default='0'), - sa.Column('added_date', sa.DateTime(), server_default=sa.func.now(), nullable=False), - sa.PrimaryKeyConstraint('id'), - sa.ForeignKeyConstraint(['collection_id'], ['collection.id'], ondelete='CASCADE'), - sa.ForeignKeyConstraint(['file_group_id'], ['file_group.id'], ondelete='CASCADE'), - sa.UniqueConstraint('collection_id', 'file_group_id', name='uq_collection_file_group') - ) - - # Create indexes for collection_item - op.create_index('idx_collection_item_collection_id', 'collection_item', ['collection_id'], unique=False) - op.create_index('idx_collection_item_file_group_id', 'collection_item', ['file_group_id'], unique=False) - op.create_index('idx_collection_item_position', 'collection_item', ['position'], unique=False) - op.create_index('idx_collection_item_collection_position', 'collection_item', ['collection_id', 'position'], unique=False) - - # Ensure table owner in non-docker environments - if not DOCKERIZED: - session.execute(sa.text('ALTER TABLE public.collection OWNER TO wrolpi')) - session.execute(sa.text('ALTER TABLE public.collection_item OWNER TO wrolpi')) - - -def downgrade(): - # Drop collection_item table and its indexes - op.drop_index('idx_collection_item_collection_position', table_name='collection_item') - op.drop_index('idx_collection_item_position', table_name='collection_item') - op.drop_index('idx_collection_item_file_group_id', table_name='collection_item') - op.drop_index('idx_collection_item_collection_id', table_name='collection_item') - op.drop_table('collection_item') - - # Drop collection table and its indexes - op.drop_index('idx_collection_total_size', table_name='collection') - op.drop_index('idx_collection_item_count', table_name='collection') - op.drop_index('idx_collection_kind', table_name='collection') - op.drop_table('collection') diff --git a/alembic/versions/69e9754d1a63_filegroup_effective_datetime.py b/alembic/versions/69e9754d1a63_filegroup_effective_datetime.py deleted file mode 100644 index d4bd15ca..00000000 --- a/alembic/versions/69e9754d1a63_filegroup_effective_datetime.py +++ /dev/null @@ -1,37 +0,0 @@ -"""FileGroup.effective_datetime - -Revision ID: 69e9754d1a63 -Revises: f7229a67e333 -Create Date: 2025-08-16 10:28:42.661108 - -""" -import os -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '69e9754d1a63' -down_revision = 'f7229a67e333' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - op.execute('ALTER TABLE file_group ADD COLUMN IF NOT EXISTS effective_datetime TIMESTAMPTZ') - - # Populating effective_datetime for existing rows - op.execute(""" - UPDATE file_group - SET effective_datetime = COALESCE(published_datetime, download_datetime) - WHERE published_datetime IS NOT NULL - OR download_datetime IS NOT NULL; - """) - - op.execute('CREATE INDEX IF NOT EXISTS file_group_effective_datetime_idx ON file_group(effective_datetime)') - - -def downgrade(): - op.execute('DROP INDEX IF EXISTS file_group_effective_datetime_idx') - op.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS effective_datetime') diff --git a/alembic/versions/6b0b249a4f81_archive_published_datetime.py b/alembic/versions/6b0b249a4f81_archive_published_datetime.py deleted file mode 100644 index 2c3a0a0c..00000000 --- a/alembic/versions/6b0b249a4f81_archive_published_datetime.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Archive published_datetime - -Revision ID: 6b0b249a4f81 -Revises: 4c8da5502c8a -Create Date: 2023-10-20 12:54:55.879180 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '6b0b249a4f81' -down_revision = '4c8da5502c8a' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE file_group ADD COLUMN author TEXT') - session.execute('ALTER TABLE file_group ADD COLUMN published_datetime TIMESTAMP WITH TIME ZONE') - session.execute('ALTER TABLE file_group ADD COLUMN published_modified_datetime TIMESTAMP WITH TIME ZONE') - - # Migrate Video.upload_date to FileGroup.published_datetime. - session.execute(''' - UPDATE file_group - SET published_datetime=upload_date - FROM video - WHERE file_group.id = video.file_group_id - ''') - - session.execute('CREATE INDEX IF NOT EXISTS file_group_author_idx ON file_group(author)') - session.execute('CREATE INDEX IF NOT EXISTS file_group_published_datetime_idx ON file_group(published_datetime)') - session.execute('CREATE INDEX IF NOT EXISTS file_group_published_modified_datetime_idx' - ' ON file_group(published_modified_datetime)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - # Migrate FileGroup.published_datetime to Video.upload_date. - session.execute(''' - UPDATE video - SET upload_date=published_datetime - FROM file_group - WHERE file_group.id = video.file_group_id - ''') - - session.execute('DROP INDEX IF EXISTS file_group_author_idx') - session.execute('DROP INDEX IF EXISTS file_group_published_datetime_idx') - session.execute('DROP INDEX IF EXISTS file_group_published_modified_datetime_idx') - - session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS author') - session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS published_datetime') - session.execute('ALTER TABLE file_group DROP COLUMN IF EXISTS published_modified_datetime') diff --git a/alembic/versions/70db5ba8763d_files_rework.py b/alembic/versions/70db5ba8763d_files_rework.py deleted file mode 100644 index d8209be0..00000000 --- a/alembic/versions/70db5ba8763d_files_rework.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Files rework. Unifying files for Videos and Archives. - -Revision ID: 70db5ba8763d -Revises: aac864072193 -Create Date: 2022-07-27 20:11:30.990396 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '70db5ba8763d' -down_revision = 'aac864072193' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - # Recreate the file table. Use file.path as the primary key. - session.execute('DROP TABLE IF EXISTS file') - session.execute('''CREATE TABLE file ( - path TEXT PRIMARY KEY, - idempotency TIMESTAMP WITH TIME ZONE, - indexed BOOLEAN DEFAULT FALSE, - mimetype TEXT, - suffix TEXT, - model TEXT, - associated BOOLEAN DEFAULT FALSE, - modification_datetime TIMESTAMP WITH TIME ZONE, - size BIGINT, - title TEXT, - a_text TEXT, - b_text TEXT, - c_text TEXT, - d_text TEXT, - textsearch tsvector GENERATED ALWAYS AS ( - setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char") - ) STORED - )''') - session.execute('CREATE INDEX file_mimetype_idx ON file USING BTREE(mimetype)') - session.execute('CREATE INDEX file_textsearch_idx ON file USING GIN(textsearch)') - session.execute('CREATE INDEX file_modification_datetime_idx ON file USING BTREE(modification_datetime)') - session.execute('CREATE INDEX file_size_idx ON file USING BTREE(size)') - session.execute('CREATE INDEX file_model_idx ON file USING BTREE(model)') - session.execute('CREATE INDEX file_idempotency_idx ON file USING BTREE(idempotency)') - session.execute('CREATE INDEX file_suffix_idx ON file USING BTREE(suffix)') - - # Videos are fundamentally different, replace the video table. - session.execute('DROP TABLE IF EXISTS video') - session.execute(''' - CREATE TABLE video ( - id BIGSERIAL PRIMARY KEY, - title TEXT, - source_id TEXT, - channel_id INTEGER REFERENCES channel(id), - upload_date TIMESTAMPTZ, - duration INTEGER, - favorite TIMESTAMPTZ, - size BIGINT, - viewed TIMESTAMPTZ, - view_count INTEGER, - url TEXT, - censored BOOLEAN DEFAULT FALSE, - validated BOOLEAN DEFAULT FALSE, - modification_datetime TIMESTAMPTZ, - video_path TEXT REFERENCES file(path), - info_json_path TEXT REFERENCES file(path), - poster_path TEXT REFERENCES file(path), - caption_path TEXT REFERENCES file(path) - )''') - session.execute(''' - CREATE INDEX video_caption_path_idx ON video(caption_path); - CREATE INDEX video_info_json_path_idx ON video(info_json_path); - CREATE INDEX video_poster_path_idx ON video(poster_path); - CREATE INDEX video_video_path_idx ON video(video_path); - CREATE INDEX video_url_idx ON video(url); - CREATE INDEX video_channel_id_idx ON video(channel_id); - ''') - - session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS idempotency') - - session.execute('DELETE FROM archive') # noqa - session.execute('DELETE FROM domains') # noqa - session.execute('DROP TABLE IF EXISTS archive') - session.execute('DROP TABLE IF EXISTS domains') - session.execute(''' - CREATE TABLE domains ( - id SERIAL PRIMARY KEY, - domain TEXT NOT NULL, - directory TEXT - )''') - session.execute(''' - CREATE TABLE archive ( - id SERIAL PRIMARY KEY, - domain_id INTEGER REFERENCES domains(id), - title TEXT, - archive_datetime timestamptz, - url TEXT, - validated BOOLEAN DEFAULT FALSE, - singlefile_path TEXT REFERENCES file(path), - readability_path TEXT REFERENCES file(path), - readability_json_path TEXT REFERENCES file(path), - readability_txt_path TEXT REFERENCES file(path), - screenshot_path TEXT REFERENCES file(path) - )''') - session.execute(''' - CREATE INDEX archive_singlefile_path ON archive(singlefile_path); - CREATE INDEX archive_readability_path ON archive(readability_path); - CREATE INDEX archive_readability_json_path ON archive(readability_json_path); - CREATE INDEX archive_readability_txt_path ON archive(readability_txt_path); - CREATE INDEX archive_screenshot_path ON archive(screenshot_path); - ''') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - DROP INDEX IF EXISTS archive_singlefile_path; - DROP INDEX IF EXISTS archive_readability_path; - DROP INDEX IF EXISTS archive_readability_json_path; - DROP INDEX IF EXISTS archive_readability_txt_path; - DROP INDEX IF EXISTS archive_screenshot_path; - ''') - - session.execute('DELETE FROM archive') # noqa - session.execute(''' - ALTER TABLE archive DROP COLUMN IF EXISTS singlefile_path; - ALTER TABLE archive DROP COLUMN IF EXISTS readability_path; - ALTER TABLE archive DROP COLUMN IF EXISTS readability_json_path; - ALTER TABLE archive DROP COLUMN IF EXISTS readability_txt_path; - ALTER TABLE archive DROP COLUMN IF EXISTS screenshot_path; - ALTER TABLE archive DROP COLUMN IF EXISTS associated; - ''') - session.execute(''' - ALTER TABLE archive ADD COLUMN IF NOT EXISTS singlefile_path TEXT; - ALTER TABLE archive ADD COLUMN IF NOT EXISTS readability_path TEXT; - ALTER TABLE archive ADD COLUMN IF NOT EXISTS readability_json_path TEXT; - ALTER TABLE archive ADD COLUMN IF NOT EXISTS readability_txt_path TEXT; - ALTER TABLE archive ADD COLUMN IF NOT EXISTS screenshot_path TEXT; - ALTER TABLE archive ADD COLUMN IF NOT EXISTS contents TEXT; - ALTER TABLE archive ADD COLUMN IF NOT EXISTS status TEXT; - ''') - session.execute('ALTER TABLE archive ALTER COLUMN domain_id SET NOT NULL') - - session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS idempotency TEXT') - - session.execute('DELETE FROM video') # noqa - session.execute(''' - ALTER TABLE video DROP COLUMN IF EXISTS caption_path; - ALTER TABLE video DROP COLUMN IF EXISTS info_json_path; - ALTER TABLE video DROP COLUMN IF EXISTS poster_path; - ALTER TABLE video DROP COLUMN IF EXISTS video_path; - ALTER TABLE video DROP COLUMN IF EXISTS video_path; - ''') - session.execute(''' - ALTER TABLE video ADD COLUMN IF NOT EXISTS idempotency TEXT; - ALTER TABLE video ADD COLUMN IF NOT EXISTS caption_path TEXT; - ALTER TABLE video ADD COLUMN IF NOT EXISTS description_path TEXT; - ALTER TABLE video ADD COLUMN IF NOT EXISTS info_json_path TEXT; - ALTER TABLE video ADD COLUMN IF NOT EXISTS poster_path TEXT; - ALTER TABLE video ADD COLUMN IF NOT EXISTS video_path TEXT; - ALTER TABLE video ADD COLUMN IF NOT EXISTS caption TEXT; - ''') - session.execute(''' - DROP INDEX IF EXISTS video_caption_path_idx; - DROP INDEX IF EXISTS video_info_json_path_idx; - DROP INDEX IF EXISTS video_poster_path_idx; - DROP INDEX IF EXISTS video_video_path_idx; - DROP INDEX IF EXISTS video_url_idx; - DROP INDEX IF EXISTS video_channel_id_idx; - ''') - session.execute(''' - ALTER TABLE video - ADD COLUMN textsearch tsvector - GENERATED ALWAYS AS ( - setweight(to_tsvector('english'::regconfig, title), 'A') || - setweight(to_tsvector('english'::regconfig, caption), 'D') - ) STORED - ''') - session.execute('CREATE INDEX IF NOT EXISTS video_textsearch_idx ON video USING GIN(textsearch)') - - session.execute('DROP INDEX IF EXISTS file_mimetype_idx') - session.execute('DROP INDEX IF EXISTS file_textsearch_idx') - session.execute('DROP INDEX IF EXISTS file_modification_datetime_idx') - session.execute('DROP INDEX IF EXISTS file_size_idx') - session.execute('DROP INDEX IF EXISTS file_model_idx') - session.execute('DROP INDEX IF EXISTS file_idempotency_idx') - session.execute('DROP INDEX IF EXISTS file_suffix_idx') - - # Restore the old file table. - session.execute('DROP TABLE IF EXISTS file') - session.execute('''CREATE TABLE file ( - id SERIAL PRIMARY KEY, - path TEXT, - title TEXT, - mimetype TEXT, - size BIGINT, - modification_datetime TIMESTAMP WITH TIME ZONE, - idempotency TEXT DEFAULT NULL, - textsearch tsvector GENERATED ALWAYS AS ( - setweight(to_tsvector('english'::regconfig, title), 'A'::"char") - ) STORED - )''') - session.execute('CREATE UNIQUE INDEX file_path_idx ON file USING BTREE(path)') - session.execute('CREATE INDEX file_mimetype_idx ON file USING BTREE(mimetype)') - session.execute('CREATE INDEX file_textsearch_idx ON file USING GIN(textsearch)') diff --git a/alembic/versions/717f36a4da94_ebook.py b/alembic/versions/717f36a4da94_ebook.py deleted file mode 100644 index f45e396a..00000000 --- a/alembic/versions/717f36a4da94_ebook.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Ebook - -Revision ID: 717f36a4da94 -Revises: d0bbe45e56c3 -Create Date: 2022-12-09 15:32:32.228539 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '717f36a4da94' -down_revision = 'd0bbe45e56c3' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - DROP INDEX IF EXISTS ebook_title_idx; - DROP INDEX IF EXISTS ebook_size_idx; - DROP INDEX IF EXISTS ebook_creator_idx; - DROP INDEX IF EXISTS ebook_ebook_path_idx; - DROP INDEX IF EXISTS ebook_cover_path_idx; - DROP TABLE IF EXISTS ebook; - ''') - - session.execute(''' - CREATE TABLE ebook ( - id SERIAL PRIMARY KEY, - size INTEGER, - title TEXT, - creator TEXT, - ebook_path TEXT REFERENCES file(path) ON DELETE CASCADE, - cover_path TEXT REFERENCES file(path) ON DELETE CASCADE - )''') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.ebook OWNER TO wrolpi') - - session.execute('CREATE INDEX ebook_title_idx ON ebook(title)') - session.execute('CREATE INDEX ebook_size_idx ON ebook(size)') - session.execute('CREATE INDEX ebook_creator_idx ON ebook(creator)') - session.execute('CREATE INDEX ebook_ebook_path_idx ON ebook(ebook_path)') - session.execute('CREATE INDEX ebook_cover_path_idx ON ebook(cover_path)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - DROP INDEX IF EXISTS ebook_title_idx; - DROP INDEX IF EXISTS ebook_size_idx; - DROP INDEX IF EXISTS ebook_creator_idx; - DROP INDEX IF EXISTS ebook_ebook_path_idx; - DROP INDEX IF EXISTS ebook_cover_path_idx; - DROP TABLE IF EXISTS ebook; - ''') diff --git a/alembic/versions/751aecf46b88_zim.py b/alembic/versions/751aecf46b88_zim.py deleted file mode 100644 index cc23d1f9..00000000 --- a/alembic/versions/751aecf46b88_zim.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Zim - -Revision ID: 751aecf46b88 -Revises: 75d34a2b442d -Create Date: 2023-06-04 19:48:41.305111 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '751aecf46b88' -down_revision = '75d34a2b442d' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - CREATE TABLE zim ( - id SERIAL PRIMARY KEY, - path TEXT NOT NULL UNIQUE, - file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL - )''') - - session.execute(''' - CREATE TABLE tag_zim ( - tag_id INTEGER REFERENCES tag(id), - zim_id INTEGER REFERENCES zim(id) ON DELETE CASCADE, - zim_entry TEXT NOT NULL, - created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - UNIQUE (tag_id, zim_id, zim_entry) - )''') - - session.execute(''' - CREATE TABLE zim_subscription ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL UNIQUE, - language TEXT NOT NULL, - download_id INTEGER NOT NULL REFERENCES download(id) ON DELETE CASCADE - ) - ''') - - # We want to index all Zim files using `zim_modeler` - session.execute("UPDATE file_group SET indexed=false WHERE primary_path ILIKE '%.zim'") - - if not DOCKERIZED: - session.execute('ALTER TABLE public.zim OWNER TO wrolpi') - session.execute('ALTER TABLE public.tag_zim OWNER TO wrolpi') - session.execute('ALTER TABLE public.zim_subscription OWNER TO wrolpi') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP TABLE IF EXISTS zim_subscription') - session.execute('DROP TABLE IF EXISTS tag_zim') - session.execute('DROP TABLE IF EXISTS zim') - session.execute("UPDATE file_group SET indexed=false WHERE primary_path ILIKE '%.zim'") diff --git a/alembic/versions/75d34a2b442d_archive_datetime_index.py b/alembic/versions/75d34a2b442d_archive_datetime_index.py deleted file mode 100644 index 539d3e73..00000000 --- a/alembic/versions/75d34a2b442d_archive_datetime_index.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Archive datetime index. - -Revision ID: 75d34a2b442d -Revises: f9928a3b6bc5 -Create Date: 2023-05-15 13:09:47.661940 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '75d34a2b442d' -down_revision = 'f9928a3b6bc5' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('CREATE INDEX archive_datetime_idx ON archive(archive_datetime)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP INDEX IF EXISTS archive_datetime_idx') diff --git a/alembic/versions/76add63ed225_json_to_jsonb.py b/alembic/versions/76add63ed225_json_to_jsonb.py deleted file mode 100644 index 643538cf..00000000 --- a/alembic/versions/76add63ed225_json_to_jsonb.py +++ /dev/null @@ -1,50 +0,0 @@ -"""json_to_jsonb - -Revision ID: 76add63ed225 -Revises: 26c5a62ec683 -Create Date: 2026-01-06 06:13:14.044016 - -""" -import os -from alembic import op -from sqlalchemy.orm import Session -from sqlalchemy import text - - -# revision identifiers, used by Alembic. -revision = '76add63ed225' -down_revision = '26c5a62ec683' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - # Convert FileGroup columns from JSON to JSONB - session.execute(text('ALTER TABLE file_group ALTER COLUMN data TYPE jsonb USING data::jsonb')) - session.execute(text('ALTER TABLE file_group ALTER COLUMN files TYPE jsonb USING files::jsonb')) - - # Convert Video column from JSON to JSONB - session.execute(text('ALTER TABLE video ALTER COLUMN ffprobe_json TYPE jsonb USING ffprobe_json::jsonb')) - - # Convert Channel column from JSON to JSONB - session.execute(text('ALTER TABLE channel ALTER COLUMN info_json TYPE jsonb USING info_json::jsonb')) - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - # Revert FileGroup columns from JSONB to JSON - session.execute(text('ALTER TABLE file_group ALTER COLUMN data TYPE json USING data::json')) - session.execute(text('ALTER TABLE file_group ALTER COLUMN files TYPE json USING files::json')) - - # Revert Video column from JSONB to JSON - session.execute(text('ALTER TABLE video ALTER COLUMN ffprobe_json TYPE json USING ffprobe_json::json')) - - # Revert Channel column from JSONB to JSON - session.execute(text('ALTER TABLE channel ALTER COLUMN info_json TYPE json USING info_json::json')) diff --git a/alembic/versions/84b6993a6091_drop_channel_next_download.py b/alembic/versions/84b6993a6091_drop_channel_next_download.py deleted file mode 100644 index f9cdea37..00000000 --- a/alembic/versions/84b6993a6091_drop_channel_next_download.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Drop channel next_download - -Revision ID: 84b6993a6091 -Revises: 8df411232c17 -Create Date: 2021-12-27 16:42:32.099796 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '84b6993a6091' -down_revision = '8df411232c17' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS next_download') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS next_download DATE') diff --git a/alembic/versions/8bdb48a486b9_video_ffprobe_json.py b/alembic/versions/8bdb48a486b9_video_ffprobe_json.py deleted file mode 100644 index e9462107..00000000 --- a/alembic/versions/8bdb48a486b9_video_ffprobe_json.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Video ffprobe json. - -Revision ID: 8bdb48a486b9 -Revises: bf704baa185d -Create Date: 2023-07-24 10:59:17.426100 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '8bdb48a486b9' -down_revision = 'bf704baa185d' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE video ADD COLUMN ffprobe_json JSON') - # Re-index all videos so the ffprobe data is fetched. - session.execute("UPDATE file_group SET indexed=false WHERE mimetype like 'video/%'") - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS ffprobe_json') diff --git a/alembic/versions/8d0d81bc9c34_channel_channel_downloads.py b/alembic/versions/8d0d81bc9c34_channel_channel_downloads.py deleted file mode 100644 index dfc00f80..00000000 --- a/alembic/versions/8d0d81bc9c34_channel_channel_downloads.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Channel.downloads / Download.channel - -Revision ID: 8d0d81bc9c34 -Revises: 00f11f309c53 -Create Date: 2024-06-25 22:13:23.885687 - -""" -import os -import pathlib - -from alembic import op -from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, Text, ForeignKey -from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Session, deferred, relationship -from sqlalchemy.orm.collections import InstrumentedList - -from wrolpi.common import ModelHelper -from wrolpi.dates import TZDateTime -from wrolpi.downloader import DownloadStatus -from wrolpi.media_path import MediaPathType - -# revision identifiers, used by Alembic. -revision = '8d0d81bc9c34' -down_revision = '00f11f309c53' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - -Base = declarative_base() - - -# The `Channel` model at the time of this migration. -class MChannel(ModelHelper, Base): - __tablename__ = 'channel' - id = Column(Integer, primary_key=True) - name = Column(String) - url = Column(String, unique=True) - directory: pathlib.Path = Column(MediaPathType) - generate_posters = Column(Boolean, default=False) - calculate_duration = Column(Boolean, default=True) - source_id = Column(String) - refreshed = Column(Boolean, default=False) - - info_json = deferred(Column(JSON)) - info_date = Column(Date) - - downloads: InstrumentedList = relationship('MDownload', primaryjoin='MDownload.channel_id==MChannel.id') - - @staticmethod - def get_by_url(url: str, session: Session = None): - if not url: - raise RuntimeError('Must provide URL to get Channel') - channel = session.query(MChannel).filter_by(url=url).one_or_none() - return channel - - def get_rss_url(self) -> str | None: - if self.url and self.source_id and 'youtube.com' in self.url: - return f'https://www.youtube.com/feeds/videos.xml?channel_id={self.source_id}' - - -# The `Download` model at the time of this migration. -class MDownload(ModelHelper, Base): - __tablename__ = 'download' - id = Column(Integer, primary_key=True) - url = Column(String, nullable=False, unique=True) - - attempts = Column(Integer, default=0) - downloader = Column(Text) - sub_downloader = Column(Text) - error = Column(Text) - frequency = Column(Integer) - info_json = Column(JSONB) - last_successful_download = Column(TZDateTime) - location = Column(Text) - next_download = Column(TZDateTime) - settings = Column(JSONB) - status = Column(String, default=DownloadStatus.new) - - channel_id = Column(Integer, ForeignKey('channel.id')) - channel = relationship('MChannel', primaryjoin='MDownload.channel_id==MChannel.id', back_populates='downloads') - - -def _link_channel_and_downloads(session: Session): - """Associate each Download with its Channel via `Download.channel_id`. - - This is a frozen copy of the linking logic as it existed when this migration was written. It deliberately does - NOT import `modules.videos.lib.link_channel_and_downloads`; that function has since been rewritten for the - Collections feature and now relies on `Download.collection_id` / `Download.destination`, neither of which exists at - this point in the migration chain. A migration must remain frozen against the schema it targets. - """ - downloads = list(session.query(MDownload).all()) - downloads_by_url = {i.url: i for i in downloads} - downloads_by_destination = {i.settings['destination']: i - for i in downloads if 'destination' in (i.settings or dict())} - channels = session.query(MChannel).all() - - need_commit = False - for channel in channels: - directory = str(channel.directory) - download = downloads_by_destination.get(directory) or downloads_by_url.get(channel.url) - if download and not download.channel_id: - download.channel_id = channel.id - need_commit = True - - # Get any Downloads for a Channel's RSS feed. - rss_url = channel.get_rss_url() - download = downloads_by_url.get(rss_url) if rss_url else None - if download and not download.channel_id: - download.channel_id = channel.id - need_commit = True - - # Associate any Download which shares a Channel's URL. - for download in downloads: - if download.url and not download.channel_id: - channel = MChannel.get_by_url(download.url, session) - if channel: - download.channel_id = channel.id - need_commit = True - - if need_commit: - session.commit() - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE download ADD CONSTRAINT download_url_unique UNIQUE (url)') - session.execute('ALTER TABLE download ADD COLUMN channel_id INTEGER REFERENCES channel(id)') - - _link_channel_and_downloads(session) - - session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS match_regex') - session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS download_frequency') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP TABLE IF EXISTS channel_download') - session.execute('ALTER TABLE download DROP COLUMN IF EXISTS channel_id') - session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS channel_id') - session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS match_regex TEXT') - session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS download_frequency INTEGER') - session.execute('ALTER TABLE download DROP CONSTRAINT IF EXISTS download_url_unique') diff --git a/alembic/versions/8df411232c17_video_indexes.py b/alembic/versions/8df411232c17_video_indexes.py deleted file mode 100644 index 6cd2c693..00000000 --- a/alembic/versions/8df411232c17_video_indexes.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Video indexes - -Revision ID: 8df411232c17 -Revises: 32d96eb9122d -Create Date: 2021-12-24 14:42:02.365100 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '8df411232c17' -down_revision = '32d96eb9122d' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('CREATE INDEX video_favorite ON video (favorite)') - session.execute('CREATE INDEX video_view_count ON video (view_count)') - session.execute('CREATE INDEX video_validated_poster ON video (validated_poster)') - session.execute('CREATE INDEX video_upload_date ON video (upload_date)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('DROP INDEX IF EXISTS video_favorite') - session.execute('DROP INDEX IF EXISTS video_view_count') - session.execute('DROP INDEX IF EXISTS video_validated_poster') - session.execute('DROP INDEX IF EXISTS video_upload_date') diff --git a/alembic/versions/90ac16f9140a_remove_file_directory_and_stem.py b/alembic/versions/90ac16f9140a_remove_file_directory_and_stem.py deleted file mode 100644 index d1cb74ef..00000000 --- a/alembic/versions/90ac16f9140a_remove_file_directory_and_stem.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Remove file directory and stem. - -Revision ID: 90ac16f9140a -Revises: 717f36a4da94 -Create Date: 2022-12-16 09:58:32.238873 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '90ac16f9140a' -down_revision = '717f36a4da94' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP INDEX IF EXISTS file_directory_idx') - session.execute('DROP INDEX IF EXISTS file_stem_idx') - session.execute('ALTER TABLE file DROP COLUMN IF EXISTS directory') - session.execute('ALTER TABLE file DROP COLUMN IF EXISTS stem') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE file ADD COLUMN directory TEXT') - session.execute('ALTER TABLE file ADD COLUMN stem TEXT') - session.execute('CREATE INDEX file_directory_idx ON file(directory)') - session.execute('CREATE INDEX file_stem_idx ON file(stem)') diff --git a/alembic/versions/930b9b7adb79_wrolpi_flags.py b/alembic/versions/930b9b7adb79_wrolpi_flags.py deleted file mode 100644 index d75c4968..00000000 --- a/alembic/versions/930b9b7adb79_wrolpi_flags.py +++ /dev/null @@ -1,39 +0,0 @@ -"""WROLPi flags - -Revision ID: 930b9b7adb79 -Revises: 90ac16f9140a -Create Date: 2022-12-20 21:02:28.975897 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '930b9b7adb79' -down_revision = '90ac16f9140a' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('''CREATE TABLE wrolpi_flag( - id SERIAL PRIMARY KEY, - refresh_complete BOOLEAN DEFAULT FALSE - )''') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.wrolpi_flag OWNER TO wrolpi') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP TABLE IF EXISTS wrolpi_flag') diff --git a/alembic/versions/9ec4c765ef8d_remove_inventories_version.py b/alembic/versions/9ec4c765ef8d_remove_inventories_version.py deleted file mode 100644 index cbd54f8d..00000000 --- a/alembic/versions/9ec4c765ef8d_remove_inventories_version.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Remove inventories version. - -Revision ID: 9ec4c765ef8d -Revises: d4e446637f18 -Create Date: 2024-09-24 16:01:41.716017 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = '9ec4c765ef8d' -down_revision = 'd4e446637f18' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP TABLE IF EXISTS inventories_version') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - CREATE TABLE inventories_version ( - version SERIAL PRIMARY KEY NOT NULL - ) - ''') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.inventories_version OWNER TO wrolpi') diff --git a/alembic/versions/a1b2c3d4e5f6_add_collection_file_format.py b/alembic/versions/a1b2c3d4e5f6_add_collection_file_format.py deleted file mode 100644 index b49491de..00000000 --- a/alembic/versions/a1b2c3d4e5f6_add_collection_file_format.py +++ /dev/null @@ -1,48 +0,0 @@ -"""add_collection_file_format - -Revision ID: a1b2c3d4e5f6 -Revises: 76add63ed225 -Create Date: 2026-01-23 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'a1b2c3d4e5f6' -down_revision = '76add63ed225' -branch_labels = None -depends_on = None - - -def upgrade(): - # Add file_format column to collection table - # This stores the file_name_format used when files were last organized - op.add_column('collection', sa.Column('file_format', sa.String(), nullable=True)) - - # Set default values from current configs so existing collections don't need reorganization - from modules.archive.lib import get_archive_downloader_config - from modules.videos.lib import get_videos_downloader_config - - connection = op.get_bind() - - # Update domain collections with archive config's file_name_format - archive_config = get_archive_downloader_config() - if archive_config.file_name_format: - connection.execute( - sa.text("UPDATE collection SET file_format = :format WHERE kind = 'domain'"), - {'format': archive_config.file_name_format} - ) - - # Update channel collections with video config's file_name_format - video_config = get_videos_downloader_config() - if video_config.file_name_format: - connection.execute( - sa.text("UPDATE collection SET file_format = :format WHERE kind = 'channel'"), - {'format': video_config.file_name_format} - ) - - -def downgrade(): - op.drop_column('collection', 'file_format') diff --git a/alembic/versions/a2b3c4d5e6f7_drop_map_file_table.py b/alembic/versions/a2b3c4d5e6f7_drop_map_file_table.py deleted file mode 100644 index 2cda1ed6..00000000 --- a/alembic/versions/a2b3c4d5e6f7_drop_map_file_table.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Drop map_file table. - -PMTiles files are now scanned directly from disk; the map_file table is no longer needed. - -Revision ID: a2b3c4d5e6f7 -Revises: e7f8a9b0c1d2 -Create Date: 2026-03-29 - -""" -from alembic import op - -# revision identifiers, used by Alembic. -revision = 'a2b3c4d5e6f7' -down_revision = 'e7f8a9b0c1d2' -branch_labels = None -depends_on = None - - -def upgrade(): - op.drop_table('map_file') - - -def downgrade(): - op.execute('''CREATE TABLE map_file ( - id SERIAL PRIMARY KEY, - path TEXT NOT NULL, - imported BOOLEAN DEFAULT FALSE, - size BIGINT - )''') diff --git a/alembic/versions/a7b1c2d3e4f5_collection_item_polymorphic.py b/alembic/versions/a7b1c2d3e4f5_collection_item_polymorphic.py deleted file mode 100644 index ceb50bae..00000000 --- a/alembic/versions/a7b1c2d3e4f5_collection_item_polymorphic.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Make collection_item polymorphic: file | zim | url items. - -CollectionItem previously only referenced a FileGroup. Playlists (kind='playlist') also need to -order Zim articles (zim_id + zim_entry, mirroring TagZimEntry) and arbitrary URLs (a link the WROLPi -browser can open, e.g. a map location). This adds item_kind plus the zim/url/title columns, relaxes -file_group_id to nullable, and adds per-kind unique constraints and a CHECK that exactly one kind's -columns are populated. Existing rows default to item_kind='file'. - -Revision ID: a7b1c2d3e4f5 -Revises: e6f7a8b9c0d1 -Create Date: 2026-06-08 - -""" -from alembic import op -import sqlalchemy as sa - -revision = 'a7b1c2d3e4f5' -down_revision = 'e6f7a8b9c0d1' -branch_labels = None -depends_on = None - -CHECK_SQL = ( - "(item_kind = 'file' AND file_group_id IS NOT NULL AND zim_id IS NULL AND url IS NULL)" - " OR (item_kind = 'zim' AND zim_id IS NOT NULL AND zim_entry IS NOT NULL" - " AND file_group_id IS NULL AND url IS NULL)" - " OR (item_kind = 'url' AND url IS NOT NULL AND file_group_id IS NULL AND zim_id IS NULL)" -) - - -def upgrade(): - op.add_column('collection_item', - sa.Column('item_kind', sa.String(), nullable=False, server_default='file')) - op.add_column('collection_item', sa.Column('zim_id', sa.Integer(), nullable=True)) - op.add_column('collection_item', sa.Column('zim_entry', sa.Text(), nullable=True)) - op.add_column('collection_item', sa.Column('url', sa.Text(), nullable=True)) - op.add_column('collection_item', sa.Column('title', sa.Text(), nullable=True)) - # `file` is no longer the only kind, so a row need not have a FileGroup. - op.alter_column('collection_item', 'file_group_id', existing_type=sa.Integer(), nullable=True) - op.create_foreign_key('collection_item_zim_id_fkey', 'collection_item', 'zim', - ['zim_id'], ['id'], ondelete='CASCADE') - op.create_unique_constraint('uq_collection_zim_entry', 'collection_item', - ['collection_id', 'zim_id', 'zim_entry']) - op.create_unique_constraint('uq_collection_url', 'collection_item', ['collection_id', 'url']) - op.create_check_constraint('ck_collection_item_kind', 'collection_item', CHECK_SQL) - - -def downgrade(): - op.drop_constraint('ck_collection_item_kind', 'collection_item', type_='check') - op.drop_constraint('uq_collection_url', 'collection_item', type_='unique') - op.drop_constraint('uq_collection_zim_entry', 'collection_item', type_='unique') - op.drop_constraint('collection_item_zim_id_fkey', 'collection_item', type_='foreignkey') - # Restore NOT NULL (valid only if no non-file rows remain). - op.alter_column('collection_item', 'file_group_id', existing_type=sa.Integer(), nullable=False) - op.drop_column('collection_item', 'title') - op.drop_column('collection_item', 'url') - op.drop_column('collection_item', 'zim_entry') - op.drop_column('collection_item', 'zim_id') - op.drop_column('collection_item', 'item_kind') diff --git a/alembic/versions/aac864072193_increase_file_size_column.py b/alembic/versions/aac864072193_increase_file_size_column.py deleted file mode 100644 index 5cb30287..00000000 --- a/alembic/versions/aac864072193_increase_file_size_column.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Increase file size column. - -Revision ID: aac864072193 -Revises: 467235201c70 -Create Date: 2022-06-22 17:34:33.635528 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'aac864072193' -down_revision = '467235201c70' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE file ALTER COLUMN size TYPE bigint') - - -def downgrade(): - pass diff --git a/alembic/versions/add_unique_collection_name_kind.py b/alembic/versions/add_unique_collection_name_kind.py deleted file mode 100644 index 10e966f5..00000000 --- a/alembic/versions/add_unique_collection_name_kind.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Add unique constraint on collection (name, kind) - -This migration: -1. Removes duplicate collections (keeping the one with most items/archives) -2. Adds a unique constraint on (name, kind) to prevent future duplicates - -Revision ID: add_unique_collection_name_kind -Revises: migrate_download_to_collection -Create Date: 2025-11-27 -""" -import os -from alembic import op -import sqlalchemy as sa -from sqlalchemy import text -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'add_unique_collection_name_kind' -down_revision = 'migrate_download_to_collection' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - print("\n" + "=" * 60) - print("Add Unique Constraint on Collection (name, kind)") - print("=" * 60 + "\n") - - # Step 1: Find and remove duplicate collections - print("Step 1: Finding duplicate collections...") - - # Find duplicates - keep the one with the lowest id (first created) - duplicates = session.execute(text(""" - SELECT name, kind, array_agg(id ORDER BY id) as ids - FROM collection - GROUP BY name, kind - HAVING COUNT(*) > 1 - """)).fetchall() - - if duplicates: - print(f"Found {len(duplicates)} sets of duplicate collections") - for name, kind, ids in duplicates: - keep_id = ids[0] # Keep the first one (lowest id) - remove_ids = ids[1:] # Remove the rest - print(f" - '{name}' ({kind}): keeping id={keep_id}, removing ids={remove_ids}") - - # Move any archives from duplicate collections to the one we're keeping - for remove_id in remove_ids: - session.execute(text(""" - UPDATE archive SET collection_id = :keep_id - WHERE collection_id = :remove_id - """), {'keep_id': keep_id, 'remove_id': remove_id}) - - # Move any downloads from duplicate collections - session.execute(text(""" - UPDATE download SET collection_id = :keep_id - WHERE collection_id = :remove_id - """), {'keep_id': keep_id, 'remove_id': remove_id}) - - # Move any collection items - session.execute(text(""" - UPDATE collection_item SET collection_id = :keep_id - WHERE collection_id = :remove_id - """), {'keep_id': keep_id, 'remove_id': remove_id}) - - # Delete the duplicate collection - session.execute(text(""" - DELETE FROM collection WHERE id = :remove_id - """), {'remove_id': remove_id}) - - session.commit() - print("Duplicates removed\n") - else: - print("No duplicate collections found\n") - - # Step 2: Add unique constraint - print("Step 2: Adding unique constraint on (name, kind)...") - op.create_unique_constraint('uq_collection_name_kind', 'collection', ['name', 'kind']) - print("Done\n") - - print("=" * 60) - print("Migration Complete") - print("=" * 60 + "\n") - - if not DOCKERIZED: - session.execute(text('ALTER TABLE public.collection OWNER TO wrolpi')) - - -def downgrade(): - op.drop_constraint('uq_collection_name_kind', 'collection', type_='unique') diff --git a/alembic/versions/af4527503bdf_add_filegroup_directory_column.py b/alembic/versions/af4527503bdf_add_filegroup_directory_column.py deleted file mode 100644 index 2e3691f7..00000000 --- a/alembic/versions/af4527503bdf_add_filegroup_directory_column.py +++ /dev/null @@ -1,192 +0,0 @@ -"""add_filegroup_directory_column - -Revision ID: af4527503bdf -Revises: 577e414f1bc4 -Create Date: 2025-12-23 19:35:18.776018 - -This migration: -1. Adds `directory` column to file_group (stores parent directory of primary_path) -2. Converts paths in `files` JSON from absolute to relative (filename only) -3. Converts paths in `data` JSON from absolute to relative (filename only) - -After this migration: -- directory: /media/wrolpi/videos/channel1 (absolute) -- primary_path: /media/wrolpi/videos/channel1/video.mp4 (absolute, unchanged for now) -- files: [{path: "video.mp4", ...}] (relative filenames) -- data: {poster_path: "video.jpg", ...} (relative filenames) - -Processing is done in batches to handle large databases (hundreds of thousands of FileGroups) -on memory-constrained devices like Raspberry Pi. -""" -import os - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.orm import Session - -import wrolpi.media_path - -# revision identifiers, used by Alembic. -revision = 'af4527503bdf' -down_revision = '577e414f1bc4' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - -# Batch size for processing - balances memory usage vs. migration speed -BATCH_SIZE = 1000 - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - # Add the directory column (nullable initially) - # Index is created in the next migration (26c5a62ec683) AFTER data is populated for efficiency. - op.add_column('file_group', sa.Column('directory', wrolpi.media_path.MediaPathType(), nullable=True)) - - # Phase 1: Populate directory from existing primary_path (batched) - # Uses regexp_replace to remove the filename (everything after the last /). - print("Phase 1: Populating directory column...") - total_updated = 0 - while True: - result = session.execute(sa.text(""" - UPDATE file_group - SET directory = regexp_replace(primary_path, '/[^/]+$', '') - WHERE id IN ( - SELECT id FROM file_group - WHERE directory IS NULL - AND primary_path IS NOT NULL - LIMIT :batch_size - ) - """), {'batch_size': BATCH_SIZE}) - session.commit() - if result.rowcount == 0: - break - total_updated += result.rowcount - print(f" Processed {total_updated} directories...") - print(f"Phase 1 complete: {total_updated} directories populated.") - - # Phase 2: Convert files JSON paths from absolute to relative (batched) - # Only processes rows where at least one path is still absolute (starts with /) - print("Phase 2: Converting files JSON to relative paths...") - total_updated = 0 - while True: - result = session.execute(sa.text(""" - UPDATE file_group - SET files = ( - SELECT jsonb_agg( - jsonb_set( - elem, - '{path}', - to_jsonb(regexp_replace(elem->>'path', '.+/', '')) - ) - ) - FROM jsonb_array_elements(files::jsonb) AS elem - ) - WHERE id IN ( - SELECT id FROM file_group - WHERE files IS NOT NULL - AND jsonb_array_length(files::jsonb) > 0 - AND EXISTS ( - SELECT 1 FROM jsonb_array_elements(files::jsonb) AS elem - WHERE elem->>'path' LIKE '/%' - ) - LIMIT :batch_size - ) - """), {'batch_size': BATCH_SIZE}) - session.commit() - if result.rowcount == 0: - break - total_updated += result.rowcount - print(f" Converted {total_updated} files arrays...") - print(f"Phase 2 complete: {total_updated} files arrays converted.") - - # Phase 3: Convert data JSON paths from absolute to relative (batched) - # Handle known path fields: video_path, info_json_path, poster_path, ebook_path, cover_path, screenshot_path - # Only processes rows where data contains at least one absolute path - print("Phase 3: Converting data JSON to relative paths...") - total_updated = 0 - while True: - result = session.execute(sa.text(""" - UPDATE file_group - SET data = ( - SELECT COALESCE( - jsonb_object_agg( - key, - CASE - -- Handle path fields that are strings starting with / - -- Use ::jsonb cast instead of to_jsonb() since value::text already has JSON quotes - WHEN key IN ('video_path', 'info_json_path', 'poster_path', 'ebook_path', 'cover_path', 'screenshot_path', - 'readability_json_path', 'readability_path', 'readability_txt_path', 'singlefile_path') - AND jsonb_typeof(value) = 'string' - AND value::text LIKE '"/%%' - THEN regexp_replace(value::text, '^".+/', '"')::jsonb - - -- Handle caption_paths which is an array of strings - WHEN key = 'caption_paths' AND jsonb_typeof(value) = 'array' - THEN ( - SELECT jsonb_agg( - CASE - WHEN jsonb_typeof(elem) = 'string' AND elem::text LIKE '"/%%' - THEN regexp_replace(elem::text, '^".+/', '"')::jsonb - ELSE elem - END - ) - FROM jsonb_array_elements(value) AS elem - ) - - -- Keep other values unchanged - ELSE value - END - ), - '{}'::jsonb - ) - FROM jsonb_each(data::jsonb) - ) - WHERE id IN ( - SELECT id FROM file_group - WHERE data IS NOT NULL - AND data::text != 'null' - AND data::text != '{}' - AND ( - -- Check if any known path field contains an absolute path - data->>'video_path' LIKE '/%' - OR data->>'info_json_path' LIKE '/%' - OR data->>'poster_path' LIKE '/%' - OR data->>'ebook_path' LIKE '/%' - OR data->>'cover_path' LIKE '/%' - OR data->>'screenshot_path' LIKE '/%' - OR data->>'readability_json_path' LIKE '/%' - OR data->>'readability_path' LIKE '/%' - OR data->>'readability_txt_path' LIKE '/%' - OR data->>'singlefile_path' LIKE '/%' - OR EXISTS ( - SELECT 1 FROM jsonb_array_elements_text( - CASE WHEN data->>'caption_paths' IS NOT NULL THEN (data->'caption_paths')::jsonb ELSE '[]'::jsonb END - ) AS elem - WHERE elem LIKE '/%' - ) - ) - LIMIT :batch_size - ) - """), {'batch_size': BATCH_SIZE}) - session.commit() - if result.rowcount == 0: - break - total_updated += result.rowcount - print(f" Converted {total_updated} data objects...") - print(f"Phase 3 complete: {total_updated} data objects converted.") - - print("Migration complete!") - - if not DOCKERIZED: - session.execute(sa.text('ALTER TABLE public.file_group OWNER TO wrolpi')) - - -def downgrade(): - # Note: Downgrade does not restore absolute paths in files/data columns. - # This would require knowing the original directory for each record. - # Index is dropped in the previous downgrade (26c5a62ec683). - op.drop_column('file_group', 'directory') diff --git a/alembic/versions/b2b9700f5048_fix_corrupted_media_paths.py b/alembic/versions/b2b9700f5048_fix_corrupted_media_paths.py deleted file mode 100644 index 47fe71b3..00000000 --- a/alembic/versions/b2b9700f5048_fix_corrupted_media_paths.py +++ /dev/null @@ -1,189 +0,0 @@ -"""Fix media paths incorrectly resolved against /opt/wrolpi. - -MediaPathType.process_bind_param() was calling .absolute() on relative paths, -which resolves against the current working directory (cwd) instead of the -media directory. In production, cwd is /opt/wrolpi, so paths like -'archive/spiegel.de' became '/opt/wrolpi/archive/spiegel.de' instead of -'/media/wrolpi/archive/spiegel.de'. - -This migration fixes any corrupted paths in the database and saves the -corrected paths to config files. - -Revision ID: b2b9700f5048 -Revises: a1b2c3d4e5f6 -Create Date: 2026-01-27 -""" -import os -import pathlib - -import yaml -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision = 'b2b9700f5048' -down_revision = 'a1b2c3d4e5f6' -branch_labels = None -depends_on = None - - -def get_relative_path(absolute_path: str, media_directory: str) -> str: - """Convert absolute path to relative path under media directory.""" - if absolute_path and absolute_path.startswith(media_directory): - return absolute_path[len(media_directory):].lstrip('/') - return absolute_path - - -def save_config_file(config: dict, file_path: pathlib.Path, width: int = 120): - """Write config dict to YAML file.""" - file_path.parent.mkdir(parents=True, exist_ok=True) - with file_path.open('wt') as fh: - yaml.dump(config, fh, width=width, sort_keys=True) - fh.flush() - os.fsync(fh.fileno()) - - -def upgrade(): - connection = op.get_bind() - - # Fix collection.directory - result = connection.execute(sa.text(""" - UPDATE collection - SET directory = REPLACE(directory, '/opt/wrolpi/', '/media/wrolpi/') - WHERE directory LIKE '/opt/wrolpi/%' - """)) - if result.rowcount: - print(f"Fixed {result.rowcount} collection directory paths") - - # Fix file_group.directory - result = connection.execute(sa.text(""" - UPDATE file_group - SET directory = REPLACE(directory, '/opt/wrolpi/', '/media/wrolpi/') - WHERE directory LIKE '/opt/wrolpi/%' - """)) - if result.rowcount: - print(f"Fixed {result.rowcount} file_group directory paths") - - # Fix file_group.primary_path - result = connection.execute(sa.text(""" - UPDATE file_group - SET primary_path = REPLACE(primary_path, '/opt/wrolpi/', '/media/wrolpi/') - WHERE primary_path LIKE '/opt/wrolpi/%' - """)) - if result.rowcount: - print(f"Fixed {result.rowcount} file_group primary_path paths") - - # Fix download.destination - result = connection.execute(sa.text(""" - UPDATE download - SET destination = REPLACE(destination, '/opt/wrolpi/', '/media/wrolpi/') - WHERE destination LIKE '/opt/wrolpi/%' - """)) - if result.rowcount: - print(f"Fixed {result.rowcount} download destination paths") - - # Save corrected paths to config files directly (bypass config framework which needs SharedContext) - try: - from wrolpi.common import get_media_directory - media_dir = str(get_media_directory()) - config_dir = get_media_directory() / 'config' - - # Save domains config - domains_result = connection.execute(sa.text(""" - SELECT id, name, kind, description, directory, file_format - FROM collection - WHERE kind = 'domain' - ORDER BY name - """)) - domains_collections = [] - for row in domains_result: - collection_config = {'name': row[1], 'kind': row[2]} - if row[3]: # description - collection_config['description'] = row[3] - if row[4]: # directory - collection_config['directory'] = get_relative_path(row[4], media_dir + '/') - if row[5]: # file_format - collection_config['file_format'] = row[5] - # Get tag name if exists - tag_result = connection.execute(sa.text(""" - SELECT t.name FROM tag t - JOIN collection c ON c.tag_id = t.id - WHERE c.id = :id - """), {'id': row[0]}).first() - collection_config['tag_name'] = tag_result[0] if tag_result else None - domains_collections.append(collection_config) - - # Read existing config to preserve version - domains_file = config_dir / 'domains.yaml' - version = 0 - if domains_file.exists(): - with domains_file.open() as f: - existing = yaml.safe_load(f) or {} - version = existing.get('version', 0) - - domains_config = {'version': version + 1, 'collections': domains_collections} - save_config_file(domains_config, domains_file) - print(f"Saved domains config with {len(domains_collections)} collections") - - # Save channels config - # Columns: 0=id, 1=name, 2=kind, 3=description, 4=directory, 5=file_format, - # 6=url, 7=source_id, 8=generate_posters, 9=calculate_duration, 10=download_missing_data - channels_result = connection.execute(sa.text(""" - SELECT c.id, c.name, c.kind, c.description, c.directory, c.file_format, - ch.url, ch.source_id, ch.generate_posters, ch.calculate_duration, - ch.download_missing_data - FROM collection c - JOIN channel ch ON ch.collection_id = c.id - WHERE c.kind = 'channel' - ORDER BY c.name - """)) - channels_list = [] - for row in channels_result: - channel_config = {'name': row[1]} - if row[4]: # directory - channel_config['directory'] = get_relative_path(row[4], media_dir + '/') - if row[6]: # url - channel_config['url'] = row[6] - if row[7]: # source_id - channel_config['source_id'] = row[7] - channel_config['generate_posters'] = row[8] - channel_config['calculate_duration'] = row[9] - channel_config['download_missing_data'] = row[10] - if row[5]: # file_format - channel_config['file_format'] = row[5] - # Get tag name if exists - tag_result = connection.execute(sa.text(""" - SELECT t.name FROM tag t - JOIN collection c ON c.tag_id = t.id - WHERE c.id = :id - """), {'id': row[0]}).first() - channel_config['tag_name'] = tag_result[0] if tag_result else None - # Get downloads for this channel's collection - downloads_result = connection.execute(sa.text(""" - SELECT url, frequency FROM download - WHERE collection_id = :collection_id - """), {'collection_id': row[0]}) - downloads = [{'url': d[0], 'frequency': d[1]} for d in downloads_result] - if downloads: - channel_config['downloads'] = downloads - channels_list.append(channel_config) - - # Read existing config to preserve version - channels_file = config_dir / 'channels.yaml' - version = 0 - if channels_file.exists(): - with channels_file.open() as f: - existing = yaml.safe_load(f) or {} - version = existing.get('version', 0) - - channels_config = {'version': version + 1, 'channels': channels_list} - save_config_file(channels_config, channels_file) - print(f"Saved channels config with {len(channels_list)} channels") - - except Exception as e: - print(f"Warning: Could not save configs: {e}") - - -def downgrade(): - # Data fix, no meaningful downgrade - pass diff --git a/alembic/versions/b2e4f6a8c1d3_drop_inventory_tables.py b/alembic/versions/b2e4f6a8c1d3_drop_inventory_tables.py deleted file mode 100644 index d000a1e1..00000000 --- a/alembic/versions/b2e4f6a8c1d3_drop_inventory_tables.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Drop inventory tables (inventory is now config-only). - -The inventory system no longer uses the database; each inventory is a YAML file at config/inventory/.yaml. -Legacy data is migrated to those files on startup (see modules/inventory/migrate.py) before these tables are dropped. - -Revision ID: b2e4f6a8c1d3 -Revises: a7b1c2d3e4f5 -Create Date: 2026-06-13 00:00:00.000000 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -revision = 'b2e4f6a8c1d3' -down_revision = 'a7b1c2d3e4f5' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - # `item` first due to its foreign key to `inventory`. - session.execute('DROP TABLE IF EXISTS public.item') - session.execute('DROP TABLE IF EXISTS public.inventory') - session.execute('DROP TABLE IF EXISTS public.inventories_version') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - CREATE TABLE inventories_version ( - version SERIAL PRIMARY KEY NOT NULL - ) - ''') - - session.execute(''' - CREATE TABLE public.inventory ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL UNIQUE, - viewed_at TIMESTAMP WITHOUT TIME ZONE, - created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - deleted_at TIMESTAMP WITHOUT TIME ZONE - )''') - - session.execute(''' - CREATE TABLE public.item ( - id SERIAL PRIMARY KEY, - inventory_id INTEGER REFERENCES inventory(id) ON DELETE CASCADE, - brand TEXT, - name TEXT, - count DECIMAL, - item_size DECIMAL, - unit TEXT, - serving INTEGER, - category TEXT, - subcategory TEXT, - expiration_date DATE, - purchase_date DATE, - created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, - deleted_at TIMESTAMP WITHOUT TIME ZONE - )''') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.inventories_version OWNER TO wrolpi') - session.execute('ALTER TABLE public.inventory OWNER TO wrolpi') - session.execute('ALTER TABLE public.item OWNER TO wrolpi') diff --git a/alembic/versions/b3c4d5e6f7a8_create_map_subscription_table.py b/alembic/versions/b3c4d5e6f7a8_create_map_subscription_table.py deleted file mode 100644 index c3cdba86..00000000 --- a/alembic/versions/b3c4d5e6f7a8_create_map_subscription_table.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Create map_subscription table. - -Revision ID: b3c4d5e6f7a8 -Revises: a2b3c4d5e6f7 -Create Date: 2026-03-29 - -""" -from alembic import op - -revision = 'b3c4d5e6f7a8' -down_revision = 'a2b3c4d5e6f7' -branch_labels = None -depends_on = None - - -def upgrade(): - op.execute('''CREATE TABLE map_subscription ( - id SERIAL PRIMARY KEY, - name TEXT UNIQUE NOT NULL, - region TEXT NOT NULL, - download_id INTEGER NOT NULL REFERENCES download(id) ON DELETE CASCADE - )''') - - -def downgrade(): - op.execute('DROP TABLE IF EXISTS map_subscription') diff --git a/alembic/versions/b43f70f369d0_remove_duplicate_channel_fields_.py b/alembic/versions/b43f70f369d0_remove_duplicate_channel_fields_.py deleted file mode 100644 index cc0da2fd..00000000 --- a/alembic/versions/b43f70f369d0_remove_duplicate_channel_fields_.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Remove duplicate Channel fields delegated to Collection - -This migration removes fields from Channel that now delegate to Collection: -- name (now property: channel.name → collection.name) -- directory (now property: channel.directory → collection.directory) -- tag_id (now property: channel.tag_id → collection.tag_id) - -These fields were already synced to Collection in previous migration. - -Revision ID: b43f70f369d0 -Revises: ba98bd360b7a -Create Date: 2025-11-19 21:48:58.488850 - -""" -import os -from alembic import op -import sqlalchemy as sa -from sqlalchemy import text -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = 'b43f70f369d0' -down_revision = 'ba98bd360b7a' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - print("\n" + "="*60) - print("Removing Duplicate Fields from Channel") - print("="*60 + "\n") - - # Step 1: Drop foreign key constraint on tag_id - print("Step 1: Dropping tag_id foreign key constraint...") - op.drop_constraint('channel_tag_id_fkey', 'channel', type_='foreignkey') - print("✓ Dropped foreign key constraint\n") - - # Step 2: Drop columns (data already in Collection) - print("Step 2: Dropping duplicate columns...") - op.drop_column('channel', 'name') - print(" ✓ Dropped channel.name") - op.drop_column('channel', 'directory') - print(" ✓ Dropped channel.directory") - op.drop_column('channel', 'tag_id') - print(" ✓ Dropped channel.tag_id\n") - - print("="*60) - print("✓ Channel Cleanup Complete") - print(" Channels now delegate name/directory/tag to Collection") - print("="*60 + "\n") - - -def downgrade(): - # Re-add the columns - op.add_column('channel', sa.Column('name', sa.String(), nullable=True)) - op.add_column('channel', sa.Column('directory', sa.String(), nullable=True)) - op.add_column('channel', sa.Column('tag_id', sa.Integer(), nullable=True)) - - # Re-add foreign key - op.create_foreign_key('channel_tag_id_fkey', 'channel', 'tag', ['tag_id'], ['id'], ondelete='CASCADE') - - # Restore data from Collection - bind = op.get_bind() - bind.execute(text(""" - UPDATE channel - SET name = collection.name, - directory = collection.directory, - tag_id = collection.tag_id - FROM collection - WHERE channel.collection_id = collection.id - """)) \ No newline at end of file diff --git a/alembic/versions/b97c1313f986_video_modification_datetime.py b/alembic/versions/b97c1313f986_video_modification_datetime.py deleted file mode 100644 index c5b478da..00000000 --- a/alembic/versions/b97c1313f986_video_modification_datetime.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Video modification_datetime - -Revision ID: b97c1313f986 -Revises: 3c33b1dae069 -Create Date: 2022-01-25 08:00:26.944183 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'b97c1313f986' -down_revision = '3c33b1dae069' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE video ADD COLUMN modification_datetime TIMESTAMPTZ') - session.execute('CREATE INDEX video_modification_datetime_idx ON video USING BRIN(modification_datetime)') - session.execute('UPDATE video SET validated=false') # get modification time for every video duration validation. - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('DROP INDEX IF EXISTS video_modification_datetime_idx') - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS modification_datetime') diff --git a/alembic/versions/ba98bd360b7a_add_collection_id_to_channel_model.py b/alembic/versions/ba98bd360b7a_add_collection_id_to_channel_model.py deleted file mode 100644 index 9766e52b..00000000 --- a/alembic/versions/ba98bd360b7a_add_collection_id_to_channel_model.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Add collection_id to Channel model - -This migration: -1. Adds collection_id column to channel table -2. Creates Collection records for existing Channels -3. Links Channels to their Collections -4. Adds foreign key constraint - -Revision ID: ba98bd360b7a -Revises: migrate_domains_to_collections -Create Date: 2025-11-19 21:10:56.472836 - -""" -import os -from alembic import op -import sqlalchemy as sa -from sqlalchemy import text -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'ba98bd360b7a' -down_revision = 'migrate_domains_to_collections' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - print("\n" + "="*60) - print("Channel → Collection Migration") - print("="*60 + "\n") - - # Step 1: Add collection_id column to channel table (nullable initially) - print("Step 1: Adding collection_id column to channel table...") - op.add_column('channel', sa.Column('collection_id', sa.Integer(), nullable=True)) - print("✓ Added collection_id column\n") - - # Step 2: Create Collection records for each Channel - print("Step 2: Creating Collection records for existing Channels...") - - # Fetch all channels - result = session.execute(text(""" - SELECT id, name, directory, tag_id - FROM channel - ORDER BY name - """)) - - channels = [] - for row in result: - channels.append({ - 'id': row[0], - 'name': row[1], - 'directory': row[2], - 'tag_id': row[3], - }) - - print(f"Found {len(channels)} Channel records to migrate") - - if channels: - for channel in channels: - channel_id = channel['id'] - channel_name = channel['name'] - - print(f" Processing Channel {channel_id}: {channel_name}") - - # Check if Collection already exists (using raw SQL to avoid ORM model mismatch) - existing = session.execute(text(""" - SELECT id FROM collection - WHERE name = :name AND kind = 'channel' - LIMIT 1 - """), {'name': channel_name}).first() - - if existing: - print(f" Collection already exists (id={existing[0]})") - collection_id = existing[0] - else: - # Create Collection using raw SQL - result = session.execute(text(""" - INSERT INTO collection (name, kind, directory, tag_id) - VALUES (:name, 'channel', :directory, :tag_id) - RETURNING id - """), { - 'name': channel_name, - 'directory': channel['directory'], - 'tag_id': channel['tag_id'], - }) - collection_id = result.first()[0] - print(f" Created Collection id={collection_id}") - - # Link Channel to Collection - session.execute( - text("UPDATE channel SET collection_id = :collection_id WHERE id = :channel_id"), - {'collection_id': collection_id, 'channel_id': channel_id} - ) - - session.commit() - print(f"✓ Created Collections and linked Channels\n") - else: - print("No channels to migrate\n") - - # Step 3: Add foreign key constraint - print("Step 3: Adding foreign key constraint...") - op.create_foreign_key( - 'fk_channel_collection_id', - 'channel', - 'collection', - ['collection_id'], - ['id'], - ondelete='CASCADE' - ) - print("✓ Added foreign key constraint\n") - - print("="*60) - print("✓ Channel → Collection Migration Complete") - print("="*60 + "\n") - - if not DOCKERIZED: - session.execute(text('ALTER TABLE public.channel OWNER TO wrolpi')) - - -def downgrade(): - # Remove foreign key constraint - op.drop_constraint('fk_channel_collection_id', 'channel', type_='foreignkey') - - # Drop collection_id column - op.drop_column('channel', 'collection_id') diff --git a/alembic/versions/bddb5e95a91e_video_textsearch_index.py b/alembic/versions/bddb5e95a91e_video_textsearch_index.py deleted file mode 100644 index 9a685157..00000000 --- a/alembic/versions/bddb5e95a91e_video_textsearch_index.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Video textsearch index - -Revision ID: bddb5e95a91e -Revises: db0a1560d923 -Create Date: 2021-12-31 18:12:00.407694 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'bddb5e95a91e' -down_revision = 'db0a1560d923' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('DROP INDEX IF EXISTS video_textsearch_idx') - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS textsearch') - session.execute(''' - ALTER TABLE video - ADD COLUMN textsearch tsvector - GENERATED ALWAYS AS ( - setweight(to_tsvector('english'::regconfig, title), 'A') || - setweight(to_tsvector('english'::regconfig, caption), 'D') - ) STORED - ''') - session.execute('CREATE INDEX IF NOT EXISTS video_textsearch_idx ON video USING GIN(textsearch)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('DROP INDEX IF EXISTS video_textsearch_idx') - session.execute(""" - ALTER TABLE video - ADD COLUMN IF NOT EXISTS textsearch tsvector - GENERATED ALWAYS AS (to_tsvector('english'::regconfig, - ((COALESCE(title, ''::text) || ' '::text) || - COALESCE(caption, ''::text)))) STORED - """) diff --git a/alembic/versions/bf704baa185d_zim_flag.py b/alembic/versions/bf704baa185d_zim_flag.py deleted file mode 100644 index d30f1303..00000000 --- a/alembic/versions/bf704baa185d_zim_flag.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Zim flag - -Revision ID: bf704baa185d -Revises: 751aecf46b88 -Create Date: 2023-07-01 20:00:35.601752 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'bf704baa185d' -down_revision = '751aecf46b88' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE wrolpi_flag ADD COLUMN outdated_zims BOOLEAN DEFAULT FALSE') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE wrolpi_flag DROP COLUMN outdated_zims') diff --git a/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py b/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py deleted file mode 100644 index 1ad2c86a..00000000 --- a/alembic/versions/c3d5f7a9b1e2_download_last_download_attempt.py +++ /dev/null @@ -1,29 +0,0 @@ -"""download_last_download_attempt - -Add the `last_download_attempt` column to the `download` table. This records the last time a Download was -dispatched (whether the attempt succeeds or fails) and is used to enforce per-domain and global daily download -limits. An index supports the "attempted since local midnight" count query. - -Revision ID: c3d5f7a9b1e2 -Revises: b2e4f6a8c1d3 -Create Date: 2026-06-28 - -""" -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision = 'c3d5f7a9b1e2' -down_revision = 'b2e4f6a8c1d3' -branch_labels = None -depends_on = None - - -def upgrade(): - op.add_column('download', sa.Column('last_download_attempt', sa.DateTime(timezone=True), nullable=True)) - op.create_index('idx_download_last_download_attempt', 'download', ['last_download_attempt']) - - -def downgrade(): - op.drop_index('idx_download_last_download_attempt', table_name='download') - op.drop_column('download', 'last_download_attempt') diff --git a/alembic/versions/c4d5e6f7a8b9_drop_map_subscription_table.py b/alembic/versions/c4d5e6f7a8b9_drop_map_subscription_table.py deleted file mode 100644 index e0d18257..00000000 --- a/alembic/versions/c4d5e6f7a8b9_drop_map_subscription_table.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Drop map_subscription table. - -Map subscriptions are now managed via settings.regions on a single Download record. - -Revision ID: c4d5e6f7a8b9 -Revises: b3c4d5e6f7a8 -Create Date: 2026-03-31 - -""" -from alembic import op - -revision = 'c4d5e6f7a8b9' -down_revision = 'b3c4d5e6f7a8' -branch_labels = None -depends_on = None - - -def upgrade(): - op.execute('DROP TABLE IF EXISTS map_subscription') - - -def downgrade(): - op.execute('''CREATE TABLE map_subscription ( - id SERIAL PRIMARY KEY, - name TEXT UNIQUE NOT NULL, - region TEXT NOT NULL, - download_id INTEGER NOT NULL REFERENCES download(id) ON DELETE CASCADE - )''') diff --git a/alembic/versions/c72d305cf11e_download_settings.py b/alembic/versions/c72d305cf11e_download_settings.py deleted file mode 100644 index 6056c36c..00000000 --- a/alembic/versions/c72d305cf11e_download_settings.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Download settings - -Revision ID: c72d305cf11e -Revises: d2eace02f5b9 -Create Date: 2022-10-14 13:00:30.333384 - -""" -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'c72d305cf11e' -down_revision = 'd2eace02f5b9' -branch_labels = None -depends_on = None - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE download ADD COLUMN settings JSONB') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE download DROP COLUMN IF EXISTS settings') diff --git a/alembic/versions/c9561bda3cf0_file_group_indexes.py b/alembic/versions/c9561bda3cf0_file_group_indexes.py deleted file mode 100644 index e67f04b5..00000000 --- a/alembic/versions/c9561bda3cf0_file_group_indexes.py +++ /dev/null @@ -1,59 +0,0 @@ -"""file_group indexes - -Revision ID: c9561bda3cf0 -Revises: 3b6918aeca4b -Create Date: 2023-03-25 20:45:00.470336 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'c9561bda3cf0' -down_revision = '3b6918aeca4b' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('CREATE INDEX file_group_mimetype_idx ON file_group(mimetype)') - session.execute('CREATE INDEX file_group_model_idx ON file_group(model)') - session.execute('CREATE INDEX file_group_modification_datetime_idx ON file_group(modification_datetime)') - session.execute('CREATE INDEX file_group_size_ix ON file_group(size)') - session.execute('CREATE INDEX file_group_textsearch_idx ON file_group USING GIN(textsearch)') - - session.execute('CREATE INDEX archive_domain_id_idx ON archive(domain_id)') - - session.execute('CREATE INDEX domains_domain_idx ON domains(domain)') - - session.execute('CREATE INDEX video_source_id_idx ON video(source_id)') - session.execute('CREATE INDEX video_upload_date_idx ON video(upload_date)') - session.execute('CREATE INDEX video_view_count_idx ON video(view_count)') - session.execute('CREATE INDEX video_viewed_idx ON video(viewed)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP INDEX IF EXISTS file_group_full_stem_idx') - session.execute('DROP INDEX IF EXISTS file_group_mimetype_idx') - session.execute('DROP INDEX IF EXISTS file_group_model_idx') - session.execute('DROP INDEX IF EXISTS file_group_modification_datetime_idx') - session.execute('DROP INDEX IF EXISTS file_group_size_ix') - - session.execute('DROP INDEX IF EXISTS archive_domain_id_idx') - - session.execute('DROP INDEX IF EXISTS domains_domain_idx') - - session.execute('DROP INDEX IF EXISTS video_source_id_idx') - session.execute('DROP INDEX IF EXISTS video_upload_date_idx') - session.execute('DROP INDEX IF EXISTS video_view_count_idx') - session.execute('DROP INDEX IF EXISTS video_viewed_idx') diff --git a/alembic/versions/d0bbe45e56c3_reindex_pdfs.py b/alembic/versions/d0bbe45e56c3_reindex_pdfs.py deleted file mode 100644 index 3c584296..00000000 --- a/alembic/versions/d0bbe45e56c3_reindex_pdfs.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Reindex pdfs - -Revision ID: d0bbe45e56c3 -Revises: dbf05c8e8dcc -Create Date: 2022-12-09 12:27:18.908533 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'd0bbe45e56c3' -down_revision = 'dbf05c8e8dcc' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute("UPDATE file SET indexed=false WHERE mimetype = 'application/pdf'") - - -def downgrade(): - pass diff --git a/alembic/versions/d2eace02f5b9_file_path_on_cascade.py b/alembic/versions/d2eace02f5b9_file_path_on_cascade.py deleted file mode 100644 index 1af5cd62..00000000 --- a/alembic/versions/d2eace02f5b9_file_path_on_cascade.py +++ /dev/null @@ -1,58 +0,0 @@ -"""file.path on cascade - -Revision ID: d2eace02f5b9 -Revises: 70db5ba8763d -Create Date: 2022-09-28 15:14:43.306819 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'd2eace02f5b9' -down_revision = '70db5ba8763d' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - ALTER TABLE video DROP CONSTRAINT video_caption_path_fkey; - ALTER TABLE video DROP CONSTRAINT video_info_json_path_fkey; - ALTER TABLE video DROP CONSTRAINT video_poster_path_fkey; - ALTER TABLE video DROP CONSTRAINT video_video_path_fkey; - ALTER TABLE video ADD FOREIGN KEY (caption_path) REFERENCES file ON DELETE CASCADE; - ALTER TABLE video ADD FOREIGN KEY (info_json_path) REFERENCES file ON DELETE CASCADE; - ALTER TABLE video ADD FOREIGN KEY (poster_path) REFERENCES file ON DELETE CASCADE; - ALTER TABLE video ADD FOREIGN KEY (video_path) REFERENCES file ON DELETE CASCADE; - ''') - - session.execute(''' - ALTER TABLE archive DROP CONSTRAINT archive_readability_json_path_fkey; - ALTER TABLE archive DROP CONSTRAINT archive_readability_path_fkey; - ALTER TABLE archive DROP CONSTRAINT archive_readability_txt_path_fkey; - ALTER TABLE archive DROP CONSTRAINT archive_screenshot_path_fkey; - ALTER TABLE archive DROP CONSTRAINT archive_singlefile_path_fkey; - ALTER TABLE archive ADD FOREIGN KEY (readability_json_path) REFERENCES file ON DELETE CASCADE; - ALTER TABLE archive ADD FOREIGN KEY (readability_path) REFERENCES file ON DELETE CASCADE; - ALTER TABLE archive ADD FOREIGN KEY (readability_txt_path) REFERENCES file ON DELETE CASCADE; - ALTER TABLE archive ADD FOREIGN KEY (screenshot_path) REFERENCES file ON DELETE CASCADE; - ALTER TABLE archive ADD FOREIGN KEY (singlefile_path) REFERENCES file ON DELETE CASCADE; - ''') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.file OWNER TO wrolpi') - session.execute('ALTER TABLE public.video OWNER TO wrolpi') - session.execute('ALTER TABLE public.domains OWNER TO wrolpi') - session.execute('ALTER TABLE public.archive OWNER TO wrolpi') - - -def downgrade(): - pass diff --git a/alembic/versions/d371333159ab_video_view_count_bigint.py b/alembic/versions/d371333159ab_video_view_count_bigint.py deleted file mode 100644 index 9674ae54..00000000 --- a/alembic/versions/d371333159ab_video_view_count_bigint.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Video view_count bigint. - -Revision ID: d371333159ab -Revises: b2b9700f5048 -Create Date: 2026-02-08 - -This migration changes the view_count column from INTEGER to BIGINT -to accommodate YouTube videos with view counts exceeding 2,147,483,647. -""" -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision = 'd371333159ab' -down_revision = 'b2b9700f5048' -branch_labels = None -depends_on = None - - -def upgrade(): - op.alter_column('video', 'view_count', - existing_type=sa.Integer(), - type_=sa.BigInteger()) - - -def downgrade(): - op.alter_column('video', 'view_count', - existing_type=sa.BigInteger(), - type_=sa.Integer()) diff --git a/alembic/versions/d4e446637f18_channel_tag.py b/alembic/versions/d4e446637f18_channel_tag.py deleted file mode 100644 index b149ccfa..00000000 --- a/alembic/versions/d4e446637f18_channel_tag.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Channel tag - -Revision ID: d4e446637f18 -Revises: 2bfb3b50b178 -Create Date: 2024-08-03 17:49:17.133086 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'd4e446637f18' -down_revision = '2bfb3b50b178' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS tag_id INTEGER REFERENCES tag(id)') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS tag_id') diff --git a/alembic/versions/d4e6f8a0c2b5_file_group_suffix.py b/alembic/versions/d4e6f8a0c2b5_file_group_suffix.py deleted file mode 100644 index 72526929..00000000 --- a/alembic/versions/d4e6f8a0c2b5_file_group_suffix.py +++ /dev/null @@ -1,69 +0,0 @@ -"""file_group_suffix - -Add the `suffix` column to the `file_group` table. This stores the lowercased suffix of the primary file -(e.g. ".bin") so searches can filter by file type using an index, rather than relying on full-text tokens. - -`suffix` is populated at ingest time alongside `mimetype` (wrolpi.files.lib._upsert_files and -FileGroup.from_paths). Existing rows are backfilled here using the same `split_path_stem_and_suffix` logic so -special multi-part suffixes like ".info.json" are stored correctly (not a naive ".json"). - -Revision ID: d4e6f8a0c2b5 -Revises: c3d5f7a9b1e2 -Create Date: 2026-07-01 - -""" -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision = 'd4e6f8a0c2b5' -down_revision = 'c3d5f7a9b1e2' -branch_labels = None -depends_on = None - -# Backfill in batches to bound memory on low-RAM devices (e.g. Raspberry Pi). -BATCH_SIZE = 5000 - - -def upgrade(): - op.add_column('file_group', sa.Column('suffix', sa.String(), nullable=True)) - op.create_index('ix_file_group_suffix', 'file_group', ['suffix']) - - # Backfill existing rows using the same suffix logic used at ingest time so the stored value matches what - # future refreshes would produce (including WROLPi special suffixes like ".info.json"). - from wrolpi.files.lib import split_path_stem_and_suffix - conn = op.get_bind() - last_id = 0 - while True: - rows = conn.execute( - sa.text('SELECT id, primary_path FROM file_group WHERE id > :last_id ORDER BY id LIMIT :limit'), - {'last_id': last_id, 'limit': BATCH_SIZE}, - ).fetchall() - if not rows: - break - - ids, suffixes = [], [] - for row_id, primary_path in rows: - suffix = (split_path_stem_and_suffix(primary_path)[1] or '').lower() or None - if suffix: - ids.append(row_id) - suffixes.append(suffix) - - if ids: - conn.execute( - sa.text(''' - UPDATE file_group AS fg - SET suffix = v.suffix - FROM (SELECT unnest(cast(:ids AS integer[])) AS id, - unnest(cast(:suffixes AS text[])) AS suffix) AS v - WHERE fg.id = v.id - '''), - {'ids': ids, 'suffixes': suffixes}, - ) - - last_id = rows[-1][0] - - -def downgrade(): - op.drop_index('ix_file_group_suffix', table_name='file_group') - op.drop_column('file_group', 'suffix') diff --git a/alembic/versions/d5e6f7a8b9c0_doc_section.py b/alembic/versions/d5e6f7a8b9c0_doc_section.py deleted file mode 100644 index a18bbd12..00000000 --- a/alembic/versions/d5e6f7a8b9c0_doc_section.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Create doc_section table for per-chapter/per-page full-text search. - -Enables deep-linking from search results into a specific EPUB chapter or PDF page, -and fixes a latent completeness bug in full-text search where FileGroup.d_text is -truncated to ~100KB (so only the first ~30 pages of a big PDF were searchable). - -Existing EPUB and PDF FileGroups are marked `indexed = FALSE` so that the regular -refresh pipeline re-runs their extractors and populates doc_section rows. No one-off -script is required. Note: the next refresh on a populated library will be heavier -than usual while every EPUB and PDF is re-extracted. - -Revision ID: d5e6f7a8b9c0 -Revises: c4d5e6f7a8b9 -Create Date: 2026-04-13 - -""" -from alembic import op - -revision = 'd5e6f7a8b9c0' -down_revision = 'c4d5e6f7a8b9' -branch_labels = None -depends_on = None - - -def upgrade(): - op.execute(''' - CREATE TABLE doc_section ( - id BIGSERIAL PRIMARY KEY, - doc_id INTEGER NOT NULL REFERENCES doc(id) ON DELETE CASCADE, - kind TEXT NOT NULL, - ordinal INTEGER NOT NULL, - label TEXT, - content TEXT, - tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('english'::regconfig, COALESCE(content, ''))) STORED - ) - ''') - op.execute('CREATE INDEX doc_section_doc_id_idx ON doc_section (doc_id, ordinal)') - op.execute('CREATE INDEX doc_section_tsv_idx ON doc_section USING GIN (tsv)') - - # Force a re-index of all EPUB and PDF file groups so the refresh pipeline - # populates the new doc_section table. - op.execute(''' - UPDATE file_group - SET indexed = FALSE - WHERE mimetype LIKE 'application/epub%' - OR mimetype = 'application/pdf' - ''') - - -def downgrade(): - op.execute('DROP TABLE IF EXISTS doc_section') - # If we drop the table while some file groups still have indexed=FALSE (the - # upgrade sets that to trigger re-extraction), the next refresh would try to - # write DocSection rows to a table that no longer exists. Restore indexed=TRUE - # for those rows so a downgrade leaves the system in a consistent state. - op.execute(''' - UPDATE file_group - SET indexed = TRUE - WHERE mimetype LIKE 'application/epub%' - OR mimetype = 'application/pdf' - ''') diff --git a/alembic/versions/d8fcdb7773c0_drop_channel_link.py b/alembic/versions/d8fcdb7773c0_drop_channel_link.py deleted file mode 100644 index 7ec3a943..00000000 --- a/alembic/versions/d8fcdb7773c0_drop_channel_link.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Drop channel.link. - -Revision ID: d8fcdb7773c0 -Revises: 2fb39c1861c7 -Create Date: 2022-04-08 12:52:02.877449 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'd8fcdb7773c0' -down_revision = '2fb39c1861c7' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE channel DROP CONSTRAINT IF EXISTS channel_link_key') - session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS link') - session.execute('DROP INDEX IF EXISTS channel_url_key') - session.execute("UPDATE channel SET url=null WHERE url=''") - session.execute('CREATE UNIQUE INDEX channel_url_key ON channel (url) WHERE url IS NOT NULL') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE channel ADD CONSTRAINT channel_link_key UNIQUE (link)') - session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS link TEXT') - # Notice we are not removing the channel_url_key constraint. diff --git a/alembic/versions/d96a04eb2785_video_have_comments.py b/alembic/versions/d96a04eb2785_video_have_comments.py deleted file mode 100644 index f29aa1af..00000000 --- a/alembic/versions/d96a04eb2785_video_have_comments.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Video.have_comments - -Revision ID: d96a04eb2785 -Revises: 365c985893b8 -Create Date: 2024-02-27 16:57:29.831210 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'd96a04eb2785' -down_revision = '365c985893b8' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE video ADD COLUMN have_comments BOOLEAN DEFAULT FALSE') - session.execute('ALTER TABLE video ADD COLUMN comments_failed BOOLEAN DEFAULT FALSE') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS have_comments') - session.execute('ALTER TABLE video DROP COLUMN IF EXISTS comments_failed') diff --git a/alembic/versions/db0a1560d923_downloader_info.py b/alembic/versions/db0a1560d923_downloader_info.py deleted file mode 100644 index 184029b3..00000000 --- a/alembic/versions/db0a1560d923_downloader_info.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Downloader info - -Revision ID: db0a1560d923 -Revises: 84b6993a6091 -Create Date: 2021-12-27 16:46:02.929119 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'db0a1560d923' -down_revision = '84b6993a6091' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS info_json JSONB') - session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS downloader TEXT') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE download DROP COLUMN IF EXISTS info_json') - session.execute('ALTER TABLE download DROP COLUMN IF EXISTS downloader') diff --git a/alembic/versions/dbf05c8e8dcc_invalidate_videos_for_orphaned_video_.py b/alembic/versions/dbf05c8e8dcc_invalidate_videos_for_orphaned_video_.py deleted file mode 100644 index 2ee13a7d..00000000 --- a/alembic/versions/dbf05c8e8dcc_invalidate_videos_for_orphaned_video_.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Invalidate videos for orphaned video files. - -Revision ID: dbf05c8e8dcc -Revises: 6489cd50c889 -Create Date: 2022-12-07 18:26:45.017144 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'dbf05c8e8dcc' -down_revision = '6489cd50c889' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('UPDATE video SET validated=false') # noqa - - -def downgrade(): - pass diff --git a/alembic/versions/dc6637ced17b_zim_auto_search.py b/alembic/versions/dc6637ced17b_zim_auto_search.py deleted file mode 100644 index b97da0fa..00000000 --- a/alembic/versions/dc6637ced17b_zim_auto_search.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Zim auto search - -Revision ID: dc6637ced17b -Revises: 1f2325523525 -Create Date: 2025-01-30 19:02:32.692428 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'dc6637ced17b' -down_revision = '1f2325523525' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE zim ADD COLUMN auto_search BOOLEAN DEFAULT TRUE') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('ALTER TABLE zim DROP COLUMN IF EXISTS auto_search') diff --git a/alembic/versions/e5f7a9c1d3b6_file_group_textsearch_abc.py b/alembic/versions/e5f7a9c1d3b6_file_group_textsearch_abc.py deleted file mode 100644 index 16207c0d..00000000 --- a/alembic/versions/e5f7a9c1d3b6_file_group_textsearch_abc.py +++ /dev/null @@ -1,43 +0,0 @@ -"""file_group_textsearch_abc - -Add the `textsearch_abc` generated tsvector column to `file_group`. It indexes only a_text (title), -b_text (author), and c_text (description) — excluding d_text (captions/document body), which is ~90% of all -text data. Searches default to this much smaller column; the existing `textsearch` column remains for -opt-in "deep" search. - -Adding a STORED generated column rewrites the table; expect a few minutes on large libraries. - -Revision ID: e5f7a9c1d3b6 -Revises: d4e6f8a0c2b5 -Create Date: 2026-07-03 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = 'e5f7a9c1d3b6' -down_revision = 'd4e6f8a0c2b5' -branch_labels = None -depends_on = None - - -def upgrade(): - conn = op.get_bind() - conn.execute(sa.text(''' - ALTER TABLE file_group - ADD COLUMN textsearch_abc tsvector - GENERATED ALWAYS AS ( - setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") - ) STORED - ''')) - conn.execute(sa.text( - 'CREATE INDEX IF NOT EXISTS file_group_textsearch_abc_idx ON file_group USING GIN(textsearch_abc)')) - - -def downgrade(): - conn = op.get_bind() - conn.execute(sa.text('DROP INDEX IF EXISTS file_group_textsearch_abc_idx')) - conn.execute(sa.text('ALTER TABLE file_group DROP COLUMN IF EXISTS textsearch_abc')) diff --git a/alembic/versions/e6f7a8b9c0d1_reindex_epubs_for_spine_order.py b/alembic/versions/e6f7a8b9c0d1_reindex_epubs_for_spine_order.py deleted file mode 100644 index ade9351a..00000000 --- a/alembic/versions/e6f7a8b9c0d1_reindex_epubs_for_spine_order.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Force EPUB re-extraction after spine-ordering fix. - -A previous version of `extract_epub` used `book.get_items_of_type(ITEM_DOCUMENT)` -to enumerate sections, which returns manifest order (not reading-spine order) and -can include non-spine documents like EPUB 3 `nav.xhtml`. The viewer resolves -deep-links via `spine.get(n)`, so any `doc_section.ordinal` written by the old -code can land users on the wrong chapter. - -The fix iterates `book.spine` directly; this migration flips `indexed = FALSE` -on every EPUB file group so the regular refresh pipeline re-extracts them with -correct spine-indexed ordinals. PDFs are untouched — their per-page ordinals -were always correct. - -Revision ID: e6f7a8b9c0d1 -Revises: d5e6f7a8b9c0 -Create Date: 2026-04-13 - -""" -from alembic import op - -revision = 'e6f7a8b9c0d1' -down_revision = 'd5e6f7a8b9c0' -branch_labels = None -depends_on = None - - -def upgrade(): - op.execute(''' - UPDATE file_group - SET indexed = FALSE - WHERE mimetype LIKE 'application/epub%' - ''') - - -def downgrade(): - # Symmetric with the parent migration's downgrade safeguard: if we re-apply - # downgrade, leave file groups marked indexed so a refresh doesn't try to - # repopulate doc_section rows that may have been dropped by the parent. - op.execute(''' - UPDATE file_group - SET indexed = TRUE - WHERE mimetype LIKE 'application/epub%' - ''') diff --git a/alembic/versions/e7f8a9b0c1d2_rename_ebook_to_doc.py b/alembic/versions/e7f8a9b0c1d2_rename_ebook_to_doc.py deleted file mode 100644 index 13be66ac..00000000 --- a/alembic/versions/e7f8a9b0c1d2_rename_ebook_to_doc.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Rename ebook table to doc and add new columns. - -Revision ID: a1b2c3d4e5f6 -Revises: d371333159ab -Create Date: 2026-03-15 15:30:00.000000 - -""" -import os - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'e7f8a9b0c1d2' -down_revision = 'd371333159ab' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False -BATCH_SIZE = 1000 - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - if not DOCKERIZED: - bind.execute(sa.text("SET ROLE 'wrolpi'")) - - # Rename ebook table to doc. - print("Migrating ebook -> doc...") - op.rename_table('ebook', 'doc') - - # Rename the index. - op.execute('ALTER INDEX IF EXISTS ebook_size_idx RENAME TO doc_size_idx') - op.execute('ALTER INDEX IF EXISTS ebook_file_group_id_key RENAME TO doc_file_group_id_key') - op.execute('ALTER INDEX IF EXISTS ebook_pkey RENAME TO doc_pkey') - - # Rename the sequence. - op.execute('ALTER SEQUENCE IF EXISTS ebook_id_seq RENAME TO doc_id_seq') - - # Add new columns. - op.add_column('doc', sa.Column('publisher', sa.String(), nullable=True)) - op.add_column('doc', sa.Column('language', sa.String(), nullable=True)) - op.add_column('doc', sa.Column('page_count', sa.Integer(), nullable=True)) - op.add_column('doc', sa.Column('subject', sa.String(), nullable=True)) - op.add_column('doc', sa.Column('description', sa.Text(), nullable=True)) - - # Widen size from Integer to BigInteger. - op.alter_column('doc', 'size', type_=sa.BigInteger(), existing_type=sa.Integer()) - - # Update file_group.model: 'ebook' -> 'doc'. - # Mark all touched file_groups as not indexed so metadata is re-extracted on next refresh. - total = 0 - while True: - result = bind.execute(sa.text( - "UPDATE file_group SET model = 'doc', indexed = false" - " WHERE id IN (SELECT id FROM file_group WHERE model = 'ebook' LIMIT :batch_size)" - ), {'batch_size': BATCH_SIZE}) - if result.rowcount == 0: - break - total += result.rowcount - print(f" Updated {total} ebook file_groups -> doc...") - session.commit() - print(f" Updated {total} ebook file_groups -> doc (done)") - - # Update file_group.model: 'pdf' -> 'doc'. - total = 0 - while True: - result = bind.execute(sa.text( - "UPDATE file_group SET model = 'doc', indexed = false" - " WHERE id IN (SELECT id FROM file_group WHERE model = 'pdf' LIMIT :batch_size)" - ), {'batch_size': BATCH_SIZE}) - if result.rowcount == 0: - break - total += result.rowcount - print(f" Updated {total} pdf file_groups -> doc...") - session.commit() - print(f" Updated {total} pdf file_groups -> doc (done)") - - # Create Doc records for existing PDFs that had no model table. - # Metadata will be extracted when the doc_modeler re-indexes these files. - total = 0 - while True: - result = bind.execute(sa.text(""" - INSERT INTO doc (file_group_id) - SELECT fg.id - FROM file_group fg - LEFT JOIN doc d ON d.file_group_id = fg.id - WHERE fg.mimetype = 'application/pdf' - AND d.id IS NULL - AND fg.primary_path IS NOT NULL - LIMIT :batch_size - """), {'batch_size': BATCH_SIZE}) - if result.rowcount == 0: - break - total += result.rowcount - print(f" Created {total} new doc records for existing PDFs...") - session.commit() - print(f" Created {total} new doc records for existing PDFs (done)") - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - if not DOCKERIZED: - bind.execute(sa.text("SET ROLE 'wrolpi'")) - - # Delete doc records that were PDFs (have no ebook equivalent). - total = 0 - while True: - result = bind.execute(sa.text(""" - DELETE FROM doc - WHERE id IN ( - SELECT d.id FROM doc d - JOIN file_group fg ON fg.id = d.file_group_id - WHERE fg.mimetype = 'application/pdf' - LIMIT :batch_size - ) - """), {'batch_size': BATCH_SIZE}) - if result.rowcount == 0: - break - total += result.rowcount - print(f" Deleted {total} PDF doc records...") - session.commit() - print(f" Deleted {total} PDF doc records (done)") - - # Revert file_group.model and re-mark as not indexed. - for old_model, mimetype_clause in [ - ('ebook', "mimetype LIKE 'application/epub%'"), - ('ebook', "mimetype = 'application/x-mobipocket-ebook'"), - ('pdf', "mimetype = 'application/pdf'"), - ]: - total = 0 - while True: - result = bind.execute(sa.text(f""" - UPDATE file_group SET model = :old_model, indexed = false - WHERE id IN ( - SELECT id FROM file_group - WHERE model = 'doc' AND {mimetype_clause} - LIMIT :batch_size - ) - """), {'old_model': old_model, 'batch_size': BATCH_SIZE}) - if result.rowcount == 0: - break - total += result.rowcount - print(f" Reverted {total} file_groups -> {old_model} ({mimetype_clause})...") - session.commit() - print(f" Reverted {total} file_groups -> {old_model} ({mimetype_clause}) (done)") - - # Revert size column type. - op.alter_column('doc', 'size', type_=sa.Integer(), existing_type=sa.BigInteger()) - - # Drop new columns. - op.drop_column('doc', 'description') - op.drop_column('doc', 'subject') - op.drop_column('doc', 'page_count') - op.drop_column('doc', 'language') - op.drop_column('doc', 'publisher') - - # Rename sequences back. - op.execute('ALTER SEQUENCE IF EXISTS doc_id_seq RENAME TO ebook_id_seq') - - # Rename indexes back. - op.execute('ALTER INDEX IF EXISTS doc_pkey RENAME TO ebook_pkey') - op.execute('ALTER INDEX IF EXISTS doc_file_group_id_key RENAME TO ebook_file_group_id_key') - op.execute('ALTER INDEX IF EXISTS doc_size_idx RENAME TO ebook_size_idx') - - # Rename table back. - op.rename_table('doc', 'ebook') diff --git a/alembic/versions/f0d0086a73d1_init_videos_tables.py b/alembic/versions/f0d0086a73d1_init_videos_tables.py deleted file mode 100644 index 8afc75a1..00000000 --- a/alembic/versions/f0d0086a73d1_init_videos_tables.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Init videos tables. - -Revision ID: f0d0086a73d1 -Revises: -Create Date: 2021-09-20 16:19:36.576217 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -revision = 'f0d0086a73d1' -down_revision = None -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('''CREATE TABLE public.channel -( - id BIGSERIAL PRIMARY KEY, - name TEXT, - link TEXT UNIQUE, - idempotency TEXT, - url TEXT, - match_regex TEXT, - directory TEXT, - info_json JSON, - info_date DATE, - skip_download_videos text[], - generate_posters BOOLEAN DEFAULT true, - calculate_duration BOOLEAN DEFAULT true, - download_frequency INTEGER, - next_download DATE -)''') - - session.execute('''CREATE TABLE public.video -( - id BIGSERIAL PRIMARY KEY, - title TEXT, - ext TEXT, - source_id TEXT, - description_path TEXT, - poster_path TEXT, - video_path TEXT, - caption_path TEXT, - info_json_path TEXT, - idempotency TEXT, - channel_id INTEGER REFERENCES channel(id) ON DELETE CASCADE, - caption TEXT, - textsearch tsvector GENERATED ALWAYS AS (to_tsvector('english'::regconfig, - ((COALESCE(title, ''::text) || ' '::text) || - COALESCE(caption, ''::text)))) STORED, - upload_date TIMESTAMP WITHOUT TIME ZONE, - duration INTEGER, - favorite TIMESTAMP WITHOUT TIME ZONE, - size BIGINT, - viewed TIMESTAMP WITHOUT TIME ZONE, - validated_poster BOOLEAN DEFAULT false, - view_count INTEGER -)''') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.channel OWNER TO wrolpi') - session.execute('ALTER TABLE public.video OWNER TO wrolpi') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP TABLE IF EXISTS channel CASCADE') - session.execute('DROP TABLE IF EXISTS video CASCADE') diff --git a/alembic/versions/f2c6326cad29_video_source_id_index.py b/alembic/versions/f2c6326cad29_video_source_id_index.py deleted file mode 100644 index b02eb2c5..00000000 --- a/alembic/versions/f2c6326cad29_video_source_id_index.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Video source_id index. - -Revision ID: f2c6326cad29 -Revises: f8c65e7e6802 -Create Date: 2022-01-10 08:27:39.082067 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'f2c6326cad29' -down_revision = 'f8c65e7e6802' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('CREATE INDEX video_source_id_idx ON video(source_id)') - session.execute('ALTER INDEX video_censored RENAME TO video_censored_idx') - session.execute('ALTER INDEX video_favorite RENAME TO video_favorite_idx') - session.execute('ALTER INDEX video_view_count RENAME TO video_view_count_idx') - session.execute('ALTER INDEX video_validated_poster RENAME TO video_validated_poster_idx') - session.execute('ALTER INDEX video_upload_date RENAME TO video_upload_date_idx') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('DROP INDEX IF EXISTS video_source_id_idx') - session.execute('ALTER INDEX video_censored_idx RENAME TO video_censored') - session.execute('ALTER INDEX video_favorite_idx RENAME TO video_favorite') - session.execute('ALTER INDEX video_view_count_idx RENAME TO video_view_count') - session.execute('ALTER INDEX video_validated_poster_idx RENAME TO video_validated_poster') - session.execute('ALTER INDEX video_upload_date_idx RENAME TO video_upload_date') diff --git a/alembic/versions/f38cfe4b5cb2_collections_feature.py b/alembic/versions/f38cfe4b5cb2_collections_feature.py deleted file mode 100644 index 39d0ed9a..00000000 --- a/alembic/versions/f38cfe4b5cb2_collections_feature.py +++ /dev/null @@ -1,68 +0,0 @@ -"""collections feature - -Revision ID: f38cfe4b5cb2 -Revises: add_unique_collection_name_kind -Create Date: 2025-11-29 16:51:34.222412 - -""" -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = 'f38cfe4b5cb2' -down_revision = 'add_unique_collection_name_kind' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('archive', 'collection_id', - existing_type=sa.INTEGER(), - nullable=True) - op.alter_column('file_group', 'mimetype', - existing_type=sa.TEXT(), - nullable=True) - op.alter_column('tag_file', 'tag_id', - existing_type=sa.INTEGER(), - nullable=False) - op.alter_column('tag_file', 'file_group_id', - existing_type=sa.BIGINT(), - nullable=False) - op.alter_column('tag_zim', 'tag_id', - existing_type=sa.INTEGER(), - nullable=False) - op.alter_column('tag_zim', 'zim_id', - existing_type=sa.INTEGER(), - nullable=False) - op.drop_index('video_upload_date_idx', table_name='video') - op.drop_column('video', 'duration') - op.drop_column('video', 'upload_date') - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('video', sa.Column('upload_date', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True)) - op.add_column('video', sa.Column('duration', sa.INTEGER(), autoincrement=False, nullable=True)) - op.create_index('video_upload_date_idx', 'video', ['upload_date'], unique=False) - op.alter_column('tag_zim', 'zim_id', - existing_type=sa.INTEGER(), - nullable=True) - op.alter_column('tag_zim', 'tag_id', - existing_type=sa.INTEGER(), - nullable=True) - op.alter_column('tag_file', 'file_group_id', - existing_type=sa.BIGINT(), - nullable=True) - op.alter_column('tag_file', 'tag_id', - existing_type=sa.INTEGER(), - nullable=True) - op.alter_column('archive', 'collection_id', - existing_type=sa.INTEGER(), - nullable=False) - op.alter_column('file_group', 'mimetype', - existing_type=sa.TEXT(), - nullable=False) - # ### end Alembic commands ### diff --git a/alembic/versions/f7229a67e333_channel_download_missing_data.py b/alembic/versions/f7229a67e333_channel_download_missing_data.py deleted file mode 100644 index 7a461b37..00000000 --- a/alembic/versions/f7229a67e333_channel_download_missing_data.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Channel.download_missing_data - -Revision ID: f7229a67e333 -Revises: dc6637ced17b -Create Date: 2025-02-19 14:02:57.952791 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'f7229a67e333' -down_revision = 'dc6637ced17b' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS download_missing_data BOOLEAN DEFAULT True') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS download_missing_data') diff --git a/alembic/versions/f8c65e7e6802_channel_refresh.py b/alembic/versions/f8c65e7e6802_channel_refresh.py deleted file mode 100644 index f7a19838..00000000 --- a/alembic/versions/f8c65e7e6802_channel_refresh.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Channel.refresh - -Revision ID: f8c65e7e6802 -Revises: 62acddc55091 -Create Date: 2022-01-07 14:28:21.787842 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'f8c65e7e6802' -down_revision = '62acddc55091' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE channel ADD COLUMN refreshed BOOL DEFAULT false') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS refreshed') diff --git a/alembic/versions/f9928a3b6bc5_file_directories.py b/alembic/versions/f9928a3b6bc5_file_directories.py deleted file mode 100644 index 5c627d60..00000000 --- a/alembic/versions/f9928a3b6bc5_file_directories.py +++ /dev/null @@ -1,45 +0,0 @@ -"""File Directories - -Revision ID: f9928a3b6bc5 -Revises: c9561bda3cf0 -Create Date: 2023-04-18 16:58:40.043115 - -""" -import os - -from alembic import op -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'f9928a3b6bc5' -down_revision = 'c9561bda3cf0' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute(''' - CREATE TABLE directory ( - path TEXT PRIMARY KEY, - name TEXT NOT NULL, - idempotency TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp - ) - ''') - - session.execute('CREATE INDEX directory_name_idx ON directory(name)') - - if not DOCKERIZED: - session.execute('ALTER TABLE public.directory OWNER TO wrolpi') - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - session.execute('DROP INDEX IF EXISTS directory_name_idx') - session.execute('DROP TABLE IF EXISTS directory') diff --git a/alembic/versions/migrate_domains_to_collections.py b/alembic/versions/migrate_domains_to_collections.py deleted file mode 100644 index 1f3fd0db..00000000 --- a/alembic/versions/migrate_domains_to_collections.py +++ /dev/null @@ -1,576 +0,0 @@ -"""Migrate Domain model to Collection model - -This migration: -1. Adds collection_id column to archive table -2. Runs the data migration to populate collections -3. Makes collection_id NOT NULL -4. Drops the old domain_id column and domains table - -Revision ID: migrate_domains_to_collections -Revises: 66407d145b76 -Create Date: 2025-10-29 - -""" -import os -import pathlib -from typing import Dict, List, Optional -from alembic import op -import sqlalchemy as sa -from sqlalchemy import text, Column, Integer, String, ForeignKey -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'migrate_domains_to_collections' -down_revision = '66407d145b76' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - -# Local model definitions for migration stability. -# These are minimal copies of the Collection and CollectionItem models -# that contain only the columns needed for this migration. -# This ensures the migration will work regardless of future changes to the actual models. -MBase = declarative_base() - - -class MCollection(MBase): - """Migration-stable Collection model with minimum required columns.""" - __tablename__ = 'collection' - - id = Column(Integer, primary_key=True) - name = Column(String, nullable=False) - kind = Column(String, nullable=False, server_default='channel', default='channel') - directory = Column(String, nullable=True) - - -class MCollectionItem(MBase): - """Migration-stable CollectionItem model with minimum required columns. - - Note: ForeignKey constraints are NOT declared here because they reference - tables (file_group) that are not in this migration's declarative base. - The actual FK constraints exist in the database schema. - """ - __tablename__ = 'collection_item' - - id = Column(Integer, primary_key=True) - collection_id = Column(Integer, nullable=False) - file_group_id = Column(Integer, nullable=False) - position = Column(Integer, nullable=False, default=0) - - -class DomainMigrationStats: - """Track migration statistics.""" - def __init__(self): - self.domains_found = 0 - self.domains_migrated = 0 - self.domains_skipped = 0 - self.collections_created = 0 - self.collections_with_directory = 0 - self.collection_items_created = 0 - self.archives_linked = 0 - self.errors: List[str] = [] - self.warnings: List[str] = [] - - def add_error(self, msg: str): - self.errors.append(msg) - print(f" ✗ ERROR: {msg}") - - def add_warning(self, msg: str): - self.warnings.append(msg) - print(f" ⚠ WARNING: {msg}") - - def print_summary(self, dry_run: bool = False): - """Print migration summary.""" - mode = "DRY-RUN" if dry_run else "ACTUAL" - print(f"\n{'='*60}") - print(f"Domain → Collection Migration Summary ({mode})") - print(f"{'='*60}") - print(f"Domains found: {self.domains_found}") - print(f"Domains migrated: {self.domains_migrated}") - print(f"Domains skipped (no archives): {self.domains_skipped}") - print(f"Collections created: {self.collections_created}") - print(f"Collections with directory: {self.collections_with_directory}") - print(f"CollectionItems created: {self.collection_items_created}") - print(f"Archives linked: {self.archives_linked}") - - if self.warnings: - print(f"\nWarnings: {len(self.warnings)}") - for warning in self.warnings: - print(f" ⚠ {warning}") - - if self.errors: - print(f"\nErrors: {len(self.errors)}") - for error in self.errors: - print(f" ✗ {error}") - - if not self.errors: - print(f"\n✓ Migration completed successfully") - else: - print(f"\n✗ Migration completed with {len(self.errors)} errors") - - print(f"{'='*60}\n") - - -def get_domain_data(session: Session) -> List[Dict]: - """Fetch all Domain records from database.""" - result = session.execute(text("SELECT id, domain, directory FROM domains ORDER BY domain")) - domains = [] - for row in result: - domains.append({ - 'id': row[0], - 'domain': row[1], - 'directory': row[2], - }) - return domains - - -def get_archives_for_domain(session: Session, domain_id: int) -> List[Dict]: - """Get all Archive IDs for a given domain.""" - result = session.execute( - text("SELECT id, file_group_id FROM archive WHERE domain_id = :domain_id"), - {'domain_id': domain_id} - ) - archives = [] - for row in result: - if row[1]: # file_group_id must not be null - archives.append({ - 'archive_id': row[0], - 'file_group_id': row[1], - }) - return archives - - -def validate_domain_name(domain: str) -> bool: - """Validate domain name format - must contain at least one dot.""" - if not domain or not isinstance(domain, str): - return False - if domain.startswith('.') or domain.endswith('.'): - return False - return '.' in domain - - -def detect_common_directory_from_archives(session: Session, domain_id: int) -> Optional[str]: - """ - Detect the common directory for all archives belonging to a domain. - - Returns the directory path as a string, or None if no common directory exists. - """ - result = session.execute( - text(""" - SELECT fg.primary_path - FROM archive a - JOIN file_group fg ON a.file_group_id = fg.id - WHERE a.domain_id = :domain_id AND fg.primary_path IS NOT NULL - """), - {'domain_id': domain_id} - ) - - paths = [pathlib.Path(row[0]) for row in result if row[0]] - - if not paths: - return None - - # Find common ancestor directory - common_dir = paths[0].parent - - for path in paths[1:]: - while common_dir != common_dir.parent: # Stop at root - try: - path.relative_to(common_dir) - break - except ValueError: - common_dir = common_dir.parent - - if common_dir == common_dir.parent: - return None # Reached root, no common directory - - return str(common_dir) - - -def check_directory_conflict(session: Session, directory: str) -> bool: - """Check if a directory is already used by an existing Collection.""" - result = session.execute( - text("SELECT id FROM collection WHERE directory = :directory"), - {'directory': directory} - ) - return result.first() is not None - - -def perform_domain_migration(session: Session, verbose: bool = True) -> DomainMigrationStats: - """ - Migrate all Domain records to Collection records. - - This function is integrated into the alembic migration. - Uses local MCollection model for migration stability, and raw SQL for - CollectionItem operations to avoid FK resolution issues with file_group. - """ - stats = DomainMigrationStats() - - # Step 1: Fetch all domains - print("Step 1: Fetching Domain records...") - domains = get_domain_data(session) - stats.domains_found = len(domains) - print(f"Found {len(domains)} Domain records") - - if not domains: - print("No domains to migrate") - return stats - - # Step 2: Create Collections from Domains - print("\nStep 2: Creating Collection records...") - domain_to_collection: Dict[int, int] = {} # domain_id -> collection_id - - for domain in domains: - domain_id = domain['id'] - domain_name = domain['domain'] - - if verbose: - print(f" Processing Domain {domain_id}: {domain_name}") - - # Validate domain name - if not validate_domain_name(domain_name): - stats.add_error(f"Invalid domain name: {repr(domain_name)} (skipping)") - continue - - # Skip domains with no archives - don't create empty collections - archives = get_archives_for_domain(session, domain_id) - if not archives: - if verbose: - print(f" Skipping domain {repr(domain_name)} - no archives") - stats.domains_skipped += 1 - continue - - # Check if Collection already exists - existing = session.query(MCollection).filter_by( - name=domain_name, - kind='domain' - ).first() - - if existing: - stats.add_warning(f"Collection already exists for domain {repr(domain_name)} (id={existing.id})") - domain_to_collection[domain_id] = existing.id - stats.domains_migrated += 1 - continue - - # Determine directory for the collection - directory = None - - # Step 1: Check if Domain.directory is set - domain_directory = domain.get('directory') - if domain_directory: - directory = domain_directory - if verbose: - print(f" Using Domain.directory: {directory}") - else: - # Step 2: Detect common directory from archive file paths - detected = detect_common_directory_from_archives(session, domain_id) - if detected: - directory = detected - if verbose: - print(f" Detected directory from archives: {directory}") - - # Step 3: Check for directory conflicts - if directory and check_directory_conflict(session, directory): - stats.add_warning( - f"Directory {repr(directory)} already used by another Collection " - f"(domain {repr(domain_name)} will have no directory)" - ) - directory = None - - # Create Collection - collection = MCollection( - name=domain_name, - kind='domain', - directory=directory, - ) - session.add(collection) - session.flush([collection]) - - domain_to_collection[domain_id] = collection.id - stats.collections_created += 1 - stats.domains_migrated += 1 - if directory: - stats.collections_with_directory += 1 - - if verbose: - if directory: - print(f" Created Collection id={collection.id} with directory={directory}") - else: - print(f" Created Collection id={collection.id} (no directory)") - - print(f"Created {stats.collections_created} Collection records") - - # Step 3: Update archive.collection_id for all Archives - print("\nStep 3: Updating archive.collection_id...") - - for domain in domains: - domain_id = domain['id'] - domain_name = domain['domain'] - - if domain_id not in domain_to_collection: - # Domain was skipped due to error - continue - - collection_id = domain_to_collection[domain_id] - - # Update all archives for this domain - result = session.execute( - text("UPDATE archive SET collection_id = :collection_id WHERE domain_id = :domain_id"), - {'collection_id': collection_id, 'domain_id': domain_id} - ) - updated_count = result.rowcount - - if verbose and updated_count > 0: - print(f" Domain {repr(domain_name)}: Updated {updated_count} archives") - - stats.archives_linked += updated_count - - print(f"Updated {stats.archives_linked} Archives with collection_id") - - # Step 3b: Handle orphan archives (those with NULL domain_id) by creating collections from their URLs - print("\nStep 3b: Handling orphan archives (NULL domain_id)...") - result = session.execute(text(""" - SELECT a.id, a.file_group_id, fg.url - FROM archive a - LEFT JOIN file_group fg ON a.file_group_id = fg.id - WHERE a.domain_id IS NULL - """)) - orphans = list(result) - if orphans: - print(f" Found {len(orphans)} orphan archives:") - - # Group orphans by domain extracted from URL - from urllib.parse import urlparse - domain_orphans = {} - for archive_id, file_group_id, url in orphans: - print(f" Archive {archive_id}: {url}") - try: - parsed = urlparse(url) - domain = parsed.netloc - if domain.startswith('www.'): - domain = domain[4:] - except Exception: - domain = 'unknown.domain' - - if domain not in domain_orphans: - domain_orphans[domain] = [] - domain_orphans[domain].append((archive_id, file_group_id)) - - # Create collections for each domain and assign orphans - for domain_name, archives in domain_orphans.items(): - print(f" Creating collection for domain: {domain_name} ({len(archives)} archives)") - - # Check if collection already exists - existing = session.execute( - text("SELECT id FROM collection WHERE name = :name AND kind = 'domain'"), - {'name': domain_name} - ).first() - - if existing: - collection_id = existing[0] - print(f" Using existing collection id={collection_id}") - else: - # Create new collection - session.execute( - text("INSERT INTO collection (name, kind) VALUES (:name, 'domain')"), - {'name': domain_name} - ) - collection_id = session.execute(text("SELECT lastval()")).scalar() - stats.collections_created += 1 - print(f" Created collection id={collection_id}") - - # Update archives with collection_id and create CollectionItems - for archive_id, file_group_id in archives: - session.execute( - text("UPDATE archive SET collection_id = :collection_id WHERE id = :archive_id"), - {'collection_id': collection_id, 'archive_id': archive_id} - ) - stats.archives_linked += 1 - - # Create CollectionItem if not exists - existing_item = session.execute( - text("SELECT id FROM collection_item WHERE collection_id = :cid AND file_group_id = :fgid"), - {'cid': collection_id, 'fgid': file_group_id} - ).first() - if not existing_item: - session.execute( - text("INSERT INTO collection_item (collection_id, file_group_id, position) VALUES (:cid, :fgid, 0)"), - {'cid': collection_id, 'fgid': file_group_id} - ) - stats.collection_items_created += 1 - - print(f" Processed {len(orphans)} orphan archives into {len(domain_orphans)} collections") - else: - print(" No orphan archives found") - - # Step 4: Create CollectionItem records - print("\nStep 4: Creating CollectionItem records...") - - for domain in domains: - domain_id = domain['id'] - domain_name = domain['domain'] - - if domain_id not in domain_to_collection: - continue - - collection_id = domain_to_collection[domain_id] - - # Get all archives for this domain - archives = get_archives_for_domain(session, domain_id) - - if not archives: - if verbose: - print(f" Domain {repr(domain_name)}: No archives found") - continue - - if verbose: - print(f" Domain {repr(domain_name)}: {len(archives)} archives") - - for idx, archive in enumerate(archives, start=1): - file_group_id = archive['file_group_id'] - archive_id = archive['archive_id'] - - # Check if CollectionItem already exists (use raw SQL to avoid FK resolution issues) - result = session.execute( - text("SELECT id FROM collection_item WHERE collection_id = :collection_id AND file_group_id = :file_group_id"), - {'collection_id': collection_id, 'file_group_id': file_group_id} - ) - existing_item = result.first() - - if existing_item: - if verbose: - print(f" Archive {archive_id} already linked to collection") - continue - - # Create CollectionItem (use raw SQL to avoid FK resolution issues) - session.execute( - text("INSERT INTO collection_item (collection_id, file_group_id, position) VALUES (:collection_id, :file_group_id, :position)"), - {'collection_id': collection_id, 'file_group_id': file_group_id, 'position': idx} - ) - stats.collection_items_created += 1 - print(f"Created {stats.collection_items_created} CollectionItem records") - - # Step 5: Skip config export during migration - # The config export requires a separate database session, but we're inside - # an alembic transaction. The application will export on startup instead. - print("\nStep 5: Skipping domains.yaml export during migration...") - print(" (Config will be exported on next application startup)") - - return stats - - -def upgrade(): - """ - Migrate Domains to Collections. - - This is a one-way migration - there is no downgrade path because: - - Domain data is transformed into Collection data - - Once migrated, the system uses Collections exclusively - - Downgrade would require reimplementing Domain model - """ - bind = op.get_bind() - session = Session(bind=bind) - - print("\n" + "="*60) - print("DOMAIN → COLLECTION MIGRATION") - print("="*60 + "\n") - - # Step 1: Add collection_id column to archive table (nullable initially) - print("Step 1: Adding collection_id column to archive table...") - op.add_column('archive', - sa.Column('collection_id', sa.Integer(), nullable=True)) - - # Add foreign key to collection table - op.create_foreign_key( - 'archive_collection_id_fkey', - 'archive', 'collection', - ['collection_id'], ['id'], - ondelete='CASCADE' - ) - print("✓ Added collection_id column\n") - - # Step 2: Run data migration script - print("Step 2: Running data migration script...") - print("This will:") - print(" - Create Collection records from Domain records") - print(" - Create CollectionItem records for Archives\n") - - try: - # Commit the schema changes before running migration - session.commit() - - # Run the data migration - stats = perform_domain_migration(session, verbose=True) - - # Print summary - stats.print_summary(dry_run=False) - - if stats.errors: - raise Exception(f"Migration completed with {len(stats.errors)} errors. See logs above.") - - except Exception as e: - print(f"\n✗ Migration failed: {e}") - raise - - # Step 3: Make collection_id NOT NULL - print("\nStep 3: Making collection_id NOT NULL...") - - # Check if any archives don't have a collection_id - result = session.execute(sa.text( - "SELECT COUNT(*) FROM archive WHERE collection_id IS NULL" - )) - null_count = result.scalar() - - if null_count > 0: - raise Exception( - f"Cannot make collection_id NOT NULL: {null_count} archives have NULL collection_id. " - f"The migration script should have assigned all archives to collections." - ) - - op.alter_column('archive', 'collection_id', nullable=False) - print("✓ collection_id is now NOT NULL\n") - - # Step 4: Drop old domain_id column and foreign key - print("Step 4: Dropping domain_id column and foreign key...") - op.drop_constraint('archive_domain_id_fkey', 'archive', type_='foreignkey') - op.drop_column('archive', 'domain_id') - print("✓ Dropped domain_id column\n") - - # Step 5: Drop domains table - print("Step 5: Dropping domains table...") - op.drop_table('domains') - print("✓ Dropped domains table\n") - - # Ensure table ownership in non-docker environments - if not DOCKERIZED: - print("Setting table ownership...") - session.execute(sa.text('ALTER TABLE public.archive OWNER TO wrolpi')) - session.commit() - - print("="*60) - print("MIGRATION COMPLETED SUCCESSFULLY") - print("="*60 + "\n") - - -def downgrade(): - """ - No downgrade path for this migration. - - This is intentional because: - 1. Domain → Collection is a one-way data transformation - 2. The Domain model is being removed from the codebase - 3. Rolling back would require: - - Recreating Domain model code - - Reversing the data transformation (Collection → Domain) - - Handling edge cases where Collections have features Domains didn't - - If you need to rollback, you should: - 1. Restore from a database backup taken before migration - 2. Re-index all files to rebuild Domain records - """ - raise NotImplementedError( - "This migration has no downgrade path. " - "To rollback, reset database and re-index files." - ) diff --git a/alembic/versions/migrate_download_channel_to_collection.py b/alembic/versions/migrate_download_channel_to_collection.py deleted file mode 100644 index 8b5be1a8..00000000 --- a/alembic/versions/migrate_download_channel_to_collection.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Migrate Download.channel_id to Download.collection_id - -This migration: -1. Adds collection_id column to download table -2. Migrates existing channel_id references to collection_id via Channel.collection_id -3. Drops channel_id column and foreign key - -Revision ID: migrate_download_to_collection -Revises: b43f70f369d0 -Create Date: 2025-11-27 -""" -import os -from alembic import op -import sqlalchemy as sa -from sqlalchemy import text -from sqlalchemy.orm import Session - -# revision identifiers, used by Alembic. -revision = 'migrate_download_to_collection' -down_revision = 'b43f70f369d0' -branch_labels = None -depends_on = None - -DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False - - -def upgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - print("\n" + "=" * 60) - print("Download channel_id -> collection_id Migration") - print("=" * 60 + "\n") - - # Step 1: Add collection_id column (nullable initially) - print("Step 1: Adding collection_id column to download table...") - op.add_column('download', sa.Column('collection_id', sa.Integer(), nullable=True)) - print("Done\n") - - # Step 2: Migrate data - lookup collection_id through channel - print("Step 2: Migrating channel_id to collection_id...") - result = session.execute(text(""" - UPDATE download - SET collection_id = channel.collection_id - FROM channel - WHERE download.channel_id = channel.id - AND download.channel_id IS NOT NULL - """)) - print(f"Updated {result.rowcount} download records\n") - - # Step 3: Add foreign key constraint - print("Step 3: Adding foreign key constraint...") - op.create_foreign_key( - 'fk_download_collection_id', - 'download', - 'collection', - ['collection_id'], - ['id'], - ondelete='SET NULL' - ) - print("Done\n") - - # Step 4: Create index for performance - print("Step 4: Creating index on collection_id...") - op.create_index('idx_download_collection_id', 'download', ['collection_id']) - print("Done\n") - - # Step 5: Drop old channel_id foreign key and column - print("Step 5: Dropping channel_id column...") - op.drop_constraint('download_channel_id_fkey', 'download', type_='foreignkey') - op.drop_column('download', 'channel_id') - print("Done\n") - - # Step 6: Fix the update_channel_minimum_frequency trigger function - # It was referencing channel_id which no longer exists - print("Step 6: Updating update_channel_minimum_frequency trigger function...") - op.execute(""" - CREATE OR REPLACE FUNCTION update_channel_minimum_frequency() - RETURNS TRIGGER AS - $$ - DECLARE - v_channel_id INTEGER; - BEGIN - -- Handle INSERT and UPDATE: use NEW record - IF (TG_OP = 'DELETE') THEN - -- For DELETE, we need to find the channel from OLD record - IF OLD.collection_id IS NOT NULL THEN - SELECT id INTO v_channel_id FROM channel WHERE collection_id = OLD.collection_id; - IF v_channel_id IS NOT NULL THEN - UPDATE channel - SET minimum_frequency = (SELECT MIN(frequency) FROM download WHERE collection_id = OLD.collection_id) - WHERE id = v_channel_id; - END IF; - END IF; - RETURN OLD; - ELSE - -- For INSERT and UPDATE, use NEW record - IF NEW.collection_id IS NOT NULL THEN - SELECT id INTO v_channel_id FROM channel WHERE collection_id = NEW.collection_id; - IF v_channel_id IS NOT NULL THEN - UPDATE channel - SET minimum_frequency = (SELECT MIN(frequency) FROM download WHERE collection_id = NEW.collection_id) - WHERE id = v_channel_id; - END IF; - END IF; - RETURN NEW; - END IF; - END; - $$ LANGUAGE plpgsql; - """) - print("Done\n") - - print("=" * 60) - print("Migration Complete") - print("=" * 60 + "\n") - - if not DOCKERIZED: - session.execute(text('ALTER TABLE public.download OWNER TO wrolpi')) - - -def downgrade(): - bind = op.get_bind() - session = Session(bind=bind) - - # Restore the old trigger function that uses channel_id - op.execute(""" - CREATE OR REPLACE FUNCTION update_channel_minimum_frequency() - RETURNS TRIGGER AS - $$ - BEGIN - UPDATE channel - SET minimum_frequency = (SELECT MIN(frequency) FROM download WHERE channel_id = NEW.channel_id) - WHERE id = NEW.channel_id; - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - """) - - # Add back channel_id column - op.add_column('download', sa.Column('channel_id', sa.Integer(), nullable=True)) - - # Migrate collection_id back to channel_id - session.execute(text(""" - UPDATE download - SET channel_id = channel.id - FROM channel - WHERE download.collection_id = channel.collection_id - AND download.collection_id IS NOT NULL - """)) - - # Add foreign key constraint - op.create_foreign_key( - 'download_channel_id_fkey', - 'download', - 'channel', - ['channel_id'], - ['id'] - ) - - # Drop collection_id - op.drop_index('idx_download_collection_id', 'download') - op.drop_constraint('fk_download_collection_id', 'download', type_='foreignkey') - op.drop_column('download', 'collection_id') diff --git a/controller/defaults.py b/controller/defaults.py index 4cb3a515..604607ee 100644 --- a/controller/defaults.py +++ b/controller/defaults.py @@ -77,14 +77,6 @@ DEFAULT_CONFIG = { "description": "Kiwix/Zim server", }, # System services - { - "name": "postgresql", - "systemd_name": "postgresql", - "port": 5432, - "viewable": False, - "use_https": False, - "description": "PostgreSQL database", - }, { "name": "caddy", "systemd_name": "caddy", diff --git a/controller/test/test_defaults.py b/controller/test/test_defaults.py index d8d49ea5..ae62b601 100644 --- a/controller/test/test_defaults.py +++ b/controller/test/test_defaults.py @@ -57,7 +57,7 @@ class TestManagedServicesConfig: def test_managed_services_count(self): """Should have expected number of managed services.""" services = DEFAULT_CONFIG["managed_services"] - assert len(services) == 12 + assert len(services) == 11 @pytest.mark.parametrize("name,expected", [ ("wrolpi-api", @@ -67,7 +67,6 @@ class TestManagedServicesConfig: ("wrolpi-app", {"port": 3000, "viewable": False}), ("wrolpi-app-dev", {"port": 3000, "viewable": False, "show_only_when_running": True}), ("caddy", {"port": 443}), - ("postgresql", {"port": 5432, "viewable": False}), ("wrolpi-upgrade", {"port": None, "viewable": False, "show_only_when_running": True}), ]) def test_service_config(self, name, expected): @@ -83,7 +82,6 @@ class TestManagedServicesConfig: ("wrolpi-kiwix", True), ("wrolpi-api", False), ("wrolpi-app", False), - ("postgresql", False), ]) def test_service_use_https(self, name, use_https): """HTTPS services should have use_https=True, others should have False.""" diff --git a/docker-compose.yml b/docker-compose.yml index a0927dcc..d8aa1d20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,21 +1,7 @@ # Development containers, these should only be used if you are developing. +# The database is SQLite inside the media directory (./test/config/wrolpi.db); no db service. version: '3' services: - db: # https://github.com/docker-library/docs/tree/master/postgres - image: postgres:12 - environment: - - POSTGRES_DB=wrolpi - - POSTGRES_PASSWORD=wrolpi - ports: - - "5432:5432" - healthcheck: - test: [ 'CMD-SHELL', 'pg_isready -U postgres' ] - interval: 10s - start_period: 30s - start_interval: 1s - volumes: - - ./pg_data:/var/lib/postgresql/data - controller: # WROLPi Controller Service build: context: . @@ -47,8 +33,6 @@ services: api: depends_on: - db: - condition: service_healthy controller: condition: service_started build: diff --git a/docker/README.md b/docker/README.md index d34b6c2f..e225362d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -11,7 +11,6 @@ git submodule update --init docker-compose build --parallel docker volume create --name=openstreetmap-data docker volume create --name=openstreetmap-rendered-tiles -docker-compose up -d db docker-compose run --rm api db upgrade docker-compose up ``` @@ -26,7 +25,6 @@ Browse to https://localhost:8443 | **api** | 8081 | Sanic API server | | **app** | (via web) | React frontend | | **controller** | 8080 | System management (FastAPI) | -| **db** | 5432 | PostgreSQL 12 | | **archive** | 8083 | SingleFile/Readability | | **zim** | (via web:8085) | Kiwix | | **help** | (via web:8086) | MkDocs documentation | diff --git a/etc/raspberrypios/wrolpi-api-dev.service b/etc/raspberrypios/wrolpi-api-dev.service index cce05a65..4e8de8ff 100644 --- a/etc/raspberrypios/wrolpi-api-dev.service +++ b/etc/raspberrypios/wrolpi-api-dev.service @@ -1,6 +1,5 @@ [Unit] Description=WROLPi Python API (Development) -Wants=postgresql.service PartOf=wrolpi.target Conflicts=wrolpi-api.service diff --git a/etc/raspberrypios/wrolpi-api.service b/etc/raspberrypios/wrolpi-api.service index ffc90159..dbc5e015 100644 --- a/etc/raspberrypios/wrolpi-api.service +++ b/etc/raspberrypios/wrolpi-api.service @@ -1,6 +1,5 @@ [Unit] Description=WROLPi Python API -Wants=postgresql.service PartOf=wrolpi.target Conflicts=wrolpi-api-dev.service diff --git a/etc/raspberrypios/wrolpi-controller.service b/etc/raspberrypios/wrolpi-controller.service index 3465cb71..37b7783d 100644 --- a/etc/raspberrypios/wrolpi-controller.service +++ b/etc/raspberrypios/wrolpi-controller.service @@ -2,7 +2,7 @@ Description=WROLPi Controller Service Documentation=https://github.com/lrnselfreliance/wrolpi After=network.target -Before=wrolpi-api.service wrolpi-app.service postgresql.service +Before=wrolpi-api.service wrolpi-app.service [Service] Type=simple diff --git a/etc/raspberrypios/wrolpi.target b/etc/raspberrypios/wrolpi.target index 4c980e1f..8551f153 100644 --- a/etc/raspberrypios/wrolpi.target +++ b/etc/raspberrypios/wrolpi.target @@ -1,3 +1,3 @@ [Unit] Description=WROLPi services group -Wants=wrolpi-controller.service wrolpi-api.service wrolpi-app.service caddy.service postgresql.service wrolpi-kiwix.service +Wants=wrolpi-controller.service wrolpi-api.service wrolpi-app.service caddy.service wrolpi-kiwix.service diff --git a/main.py b/main.py index 979e53f1..4e6d09c0 100755 --- a/main.py +++ b/main.py @@ -43,8 +43,9 @@ def db_main(args): """ from alembic.config import Config from alembic import command - from wrolpi.db import uri + from wrolpi.db import get_db_uri + uri = get_db_uri() config = Config(PROJECT_DIR / 'alembic.ini') # Overwrite the Alembic config, the is usually necessary when running in a docker container. config.set_main_option('sqlalchemy.url', uri) @@ -190,6 +191,14 @@ async def main_process_startup(app: Sanic): except Exception as e: logger.error('Failed to create initial config files', exc_info=e) + try: + # Create/migrate the SQLite database (if the media directory is available). If the media + # directory is mounted later, `perpetual_check_db_is_up_worker` retries this. + from wrolpi import db_bootstrap + db_bootstrap.ensure_db() + except Exception as e: + logger.error('Failed to create/migrate the database', exc_info=e) + @api_app.listener('after_server_start') # FileConfigs need to be initialized first. async def initialize_configs(app: Sanic): @@ -324,6 +333,13 @@ async def perpetual_check_log_level(): @perpetual_signal(sleep=10) async def perpetual_check_db_is_up_worker(): try: + if not flags.db_up.is_set(): + # The DB may not exist yet (media drive mounted after startup); try to create/migrate + # it. Only one worker bootstraps at a time. + from wrolpi import db_bootstrap + with db_bootstrap.bootstrap_lock() as acquired: + if acquired: + db_bootstrap.ensure_db() flags.check_db_is_up() flags.init_flags() except Exception as e: diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 778e1de2..7246aaf0 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -20,6 +20,7 @@ from sqlalchemy.orm import Session from modules.archive.models import Archive from wrolpi import dates +from wrolpi import fts from wrolpi.cmd import READABILITY_BIN, run_command from wrolpi.collections import Collection from wrolpi.common import get_media_directory, get_relative_to_media_directory, logger, extract_domain, \ @@ -120,6 +121,7 @@ class DomainsConfig(ConfigFile): name = collection_data.get('name') if not name: logger.error(f'Domain collection at index {idx} has no name, skipping') + self.import_skipped += 1 continue # Validate domain name format before batching @@ -128,6 +130,7 @@ class DomainsConfig(ConfigFile): f"Domain collection at index {idx} has invalid name '{name}' " f"(must contain at least one '.' and not start/end with '.'), skipping" ) + self.import_skipped += 1 continue # Ensure kind is 'domain' @@ -140,6 +143,7 @@ class DomainsConfig(ConfigFile): valid_data_list.append(collection_data) except Exception as e: logger.error(f'Failed to validate domain collection at index {idx}', exc_info=e) + self.import_skipped += 1 continue # Batch create/update collections @@ -1344,10 +1348,10 @@ def get_domain(domain_id: int) -> dict: # Get archive statistics for this domain with get_db_curs() as curs: stmt = ''' - SELECT COUNT(a.id) AS url_count, SUM(fg.size)::BIGINT AS size + SELECT COUNT(a.id) AS url_count, SUM(fg.size) AS size FROM archive a LEFT JOIN file_group fg on fg.id = a.file_group_id - WHERE a.collection_id = %(domain_id)s + WHERE a.collection_id = :domain_id ''' curs.execute(stmt, {'domain_id': domain_id}) stats = dict(curs.fetchone()) @@ -1377,20 +1381,6 @@ ARCHIVE_ORDERS = { 'viewed': 'fg.viewed ASC', '-viewed': 'fg.viewed DESC', } -ORDER_GROUP_BYS = { - 'published_datetime': 'fg.effective_datetime', - '-published_datetime': 'fg.effective_datetime', - 'published_modified_datetime': 'fg.published_modified_datetime, fg.published_datetime, fg.download_datetime', - '-published_modified_datetime': 'fg.published_modified_datetime, fg.published_datetime, fg.download_datetime', - 'download_datetime': 'fg.download_datetime', - '-download_datetime': 'fg.download_datetime', - 'rank': 'fg.published_datetime, fg.download_datetime', - '-rank': 'fg.published_datetime, fg.download_datetime', - 'size': 'fg.size, fg.published_datetime, fg.primary_path', - '-size': 'fg.size, fg.published_datetime, fg.primary_path', - 'viewed': 'fg.viewed', - '-viewed': 'fg.viewed', -} NO_NULL_ORDERS = { 'viewed': 'fg.viewed IS NOT NULL', '-viewed': 'fg.viewed IS NOT NULL', @@ -1404,26 +1394,40 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order -> Tuple[List[dict], int]: # Always filter FileGroups to Archives. wheres = [] - group_by = 'fg.effective_datetime, a.file_group_id' - params = dict(search_str=search_str, offset=int(offset), limit=int(limit)) + # Explicit JSON nulls bypass the schema defaults; SQLite rejects LIMIT/OFFSET NULL. + params = dict(search_str=search_str, offset=int(offset) if offset else 0, limit=int(limit) if limit else 20) order_by = ARCHIVE_ORDERS['-published_datetime'] select_columns = '' - if search_str: - # A search_str was provided by the user, modify the query to filter by it. - # `textsearch` includes d_text (page contents); `textsearch_abc` is much smaller and faster. - ts_col = 'textsearch' if deep else 'textsearch_abc' - select_columns = f'ts_rank(fg.{ts_col}, websearch_to_tsquery(%(search_str)s)) AS rank,' \ - ' COUNT(*) OVER() AS total' - wheres.append(f'fg.{ts_col} @@ websearch_to_tsquery(%(search_str)s)') - params['search_str'] = search_str - group_by = f'{group_by}, rank' + fts_join = '' + headline_columns = '' + # The fast (deep=False) search only matches the a/b/c FTS columns; deep also matches d + # (page contents). `file_group_search` returns None when the search string is unusable, in + # which case we behave as if no search was requested. + file_group_search = fts.file_group_search(search_str, deep=deep) if search_str else None + if file_group_search: + # A search_str was provided by the user, modify the query to filter by it. The rank and + # snippets come from a subquery because FTS5 auxiliary functions (snippet) cannot appear + # in a query that also uses window functions (COUNT(*) OVER()). + headline_selects = fts.file_group_headline_selects() if headline else '' + fts_join = f'''JOIN ( + SELECT fts.rowid AS id, {file_group_search.rank_select}{headline_selects} + FROM file_group_fts fts + WHERE {file_group_search.where} + ) fts ON fts.id = fg.id''' + select_columns = 'fts.ts_rank AS ts_rank' # positive, higher is better. + params.update(file_group_search.params) + if headline: + # b/c/d headlines come from the FTS index. `title` is not an FTS column; select it + # plainly and highlight it in Python below. + headline_columns = ''', + fts.b_headline, fts.c_headline, fts.d_headline, + fg.title AS title_headline''' if order: try: order_by = ARCHIVE_ORDERS[order] - group_by = f'{group_by}, {ORDER_GROUP_BYS[order]}' except KeyError: raise InvalidOrderBy(f'Invalid order byy {order}') if order in NO_NULL_ORDERS: @@ -1431,41 +1435,40 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order wheres, params = tag_append_sub_select_where(wheres, params, tag_names) - if search_str and headline: - headline = ''', - ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline", - ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline", - ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline", - ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"''' - group_by = f'{group_by}, title_headline, b_headline, c_headline, d_headline' - else: - headline = '' - if domain: params['domain'] = domain # Use LIMIT 1 to handle potential duplicate domain collections with the same name wheres.append( - "a.collection_id = (select id from collection where collection.name = %(domain)s and collection.kind = 'domain' LIMIT 1)") + "a.collection_id = (select id from collection where collection.name = :domain and collection.kind = 'domain' LIMIT 1)") - select_columns = f", {select_columns}" if select_columns else "" + select_columns = f"{select_columns}," if select_columns else "" wheres = '\n AND '.join(wheres) where = f'WHERE\n{wheres}' if wheres else '' stmt = f''' SELECT a.file_group_id AS id, -- always get `file_group.id` for `handle_file_group_search_results` - COUNT(*) OVER() AS total {select_columns} - {headline} + COUNT(*) OVER() AS total + {headline_columns} FROM archive a LEFT JOIN file_group fg ON fg.id = a.file_group_id + {fts_join} {where} - GROUP BY {group_by} ORDER BY {order_by} - OFFSET %(offset)s LIMIT %(limit)s + LIMIT :limit OFFSET :offset '''.strip() - logger.debug(stmt, params) + logger.debug(f'{stmt} {params}') results, total = handle_file_group_search_results(stmt, params) + + if file_group_search and headline and results: + # Highlight the plain titles like the FTS snippets above (Postgres used ts_headline). + titles = [i.get('title_headline') for i in results] + highlighted = fts.headline_texts(titles, search_str) + for result, title, (title_headline, _) in zip(results, titles, highlighted): + if title is not None: + result['title_headline'] = title_headline + return results, total @@ -1617,39 +1620,41 @@ async def get_statistics(): curs.execute(''' SELECT -- total archives - COUNT(a.id) AS "archives", + COUNT(a.id) AS "archives", -- total archives downloaded over the past week/month/year - COUNT(a.id) FILTER (WHERE fg.download_datetime >= current_date - interval '1 week') AS "week", + COUNT(a.id) FILTER (WHERE fg.download_datetime >= datetime('now', '-7 days')) AS "week", COUNT(a.id) - FILTER (WHERE fg.download_datetime >= current_date - interval '1 month') AS "month", - COUNT(a.id) FILTER (WHERE fg.download_datetime >= current_date - interval '1 year') AS "year", + FILTER (WHERE fg.download_datetime >= datetime('now', '-1 month')) AS "month", + COUNT(a.id) FILTER (WHERE fg.download_datetime >= datetime('now', '-1 year')) AS "year", -- sum of all archive file sizes - COALESCE(SUM(fg.size), 0)::BIGINT AS "sum_size", + COALESCE(SUM(fg.size), 0) AS "sum_size", -- largest archive - COALESCE(MAX(fg.size), 0) AS "max_size" + COALESCE(MAX(fg.size), 0) AS "max_size" FROM archive a LEFT JOIN file_group fg on a.file_group_id = fg.id ''') archive_stats = dict(curs.fetchone()) - # Get the total archives downloaded every month for the past two years. + # Get the total archives downloaded every month for the past two years (excluding the + # current month, like the old generate_series query). curs.execute(''' - SELECT DATE_TRUNC('month', months.a), - COUNT(archive.id)::BIGINT, - SUM(size)::BIGINT AS "size" - FROM generate_series( - date_trunc('month', current_date) - interval '2 years', - date_trunc('month', current_date) - interval '1 month', - '1 month'::interval) AS months(a), - archive + SELECT strftime('%Y-%m-01 00:00:00', fg.download_datetime) AS "date_trunc", + COUNT(archive.id) AS "count", + SUM(fg.size) AS "size" + FROM archive LEFT JOIN file_group fg on archive.file_group_id = fg.id - WHERE fg.download_datetime >= date_trunc('month', months.a) - AND fg.download_datetime < date_trunc('month', months.a) + interval '1 month' - AND fg.download_datetime IS NOT NULL + WHERE fg.download_datetime IS NOT NULL + AND fg.download_datetime >= datetime('now', 'start of month', '-24 months') + AND fg.download_datetime < datetime('now', 'start of month') GROUP BY 1 ORDER BY 1 ''') - monthly_archives = [dict(i) for i in curs.fetchall()] + monthly_archives = [] + for row in curs.fetchall(): + row = dict(row) + # Convert the month strings back to datetimes so the JSON response shape is unchanged. + row['date_trunc'] = datetime.strptime(row['date_trunc'], '%Y-%m-%d %H:%M:%S') + monthly_archives.append(row) historical_stats = dict(monthly_archives=monthly_archives) historical_stats['average_count'] = (sum(i['count'] for i in monthly_archives) // len(monthly_archives)) \ diff --git a/modules/archive/test/test_api.py b/modules/archive/test/test_api.py index c92df3d1..684c6656 100644 --- a/modules/archive/test/test_api.py +++ b/modules/archive/test/test_api.py @@ -75,9 +75,9 @@ async def test_archives_search(test_session, archive_directory, archive_factory, data = {'search_str': 'baz', 'domain': 'example.org', 'deep': True} await check_results(async_client, data, [3, ]) - # 1's title contains "my", this is ignored by Postgres. + # 1's title contains "my". Postgres ignored this stopword; FTS5 has no stopwords, so it matches. data = {'search_str': 'my', 'deep': True} - await check_results(async_client, data, []) + await check_results(async_client, data, [1, ]) # 3's title contains "third", found by the default (fast) search and the deep search. data = {'search_str': 'third'} diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 625782d0..6d1fc59d 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -1053,8 +1053,9 @@ def test_collection_unique_name_kind_constraint(test_directory, test_session, ar with pytest.raises(IntegrityError) as exc_info: test_session.commit() - # Verify it's the unique constraint violation - assert 'uq_collection_name_kind' in str(exc_info.value) + # Verify it's the unique constraint violation. SQLite reports the columns, not the + # constraint name (Postgres reported 'uq_collection_name_kind'). + assert 'collection.name, collection.kind' in str(exc_info.value) test_session.rollback() diff --git a/modules/docs/lib.py b/modules/docs/lib.py index 79060638..657b0abd 100644 --- a/modules/docs/lib.py +++ b/modules/docs/lib.py @@ -4,9 +4,10 @@ import pathlib import re from typing import List, Optional -from sqlalchemy import asc, desc, func, nullslast, text +from sqlalchemy import asc, desc, func, nullslast, text, column, bindparam, Integer, Float from sqlalchemy.orm import Session +from wrolpi import fts from wrolpi.collections.models import Collection from wrolpi.collections.types import collection_type_registry from wrolpi.db import get_db_session @@ -351,14 +352,21 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime from .models import Doc from wrolpi.files.models import FileGroup - # `textsearch` includes d_text (document contents); `textsearch_abc` is much smaller and faster. - ts_col = FileGroup.textsearch if deep else FileGroup.textsearch_abc + # Deep search matches all FTS columns (including d_text, the document contents); the default matches only + # the much smaller a/b/c columns. + match = fts.translate_websearch(search_str, None if deep else fts.ABC_COLUMNS) if search_str else None with get_db_session() as session: query = session.query(FileGroup).join(Doc, Doc.file_group_id == FileGroup.id) - if search_str: - query = query.filter(ts_col.op('@@')(func.websearch_to_tsquery(search_str))) + fts_sq = None + if match is not None: + fts_sq = text('SELECT rowid AS id, -rank AS ts_rank' + ' FROM file_group_fts WHERE file_group_fts MATCH :fts_match') \ + .bindparams(fts_match=match) \ + .columns(column('id', Integer), column('ts_rank', Float)) \ + .alias('fts') + query = query.join(fts_sq, fts_sq.c.id == FileGroup.id) if author: query = query.filter(FileGroup.b_text.ilike(f'%{author}%')) @@ -383,9 +391,9 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime total = query.count() # Ordering. - if order_by == 'rank' and search_str: + if order_by == 'rank' and fts_sq is not None: query = query.order_by( - desc(func.ts_rank(ts_col, func.websearch_to_tsquery(search_str))), + desc(fts_sq.c.ts_rank), desc(FileGroup.id), ) elif order_by == 'published_datetime': @@ -416,26 +424,35 @@ def _fetch_section_hints(session, file_group_ids, search_str): Returns a mapping of file_group_id -> {kind, ordinal, label, snippet}. """ - from .models import Doc, DocSection - if not file_group_ids: return {} + match = fts.translate_websearch(search_str) + if match is None: + return {} + + # FTS5 auxiliary functions (snippet, rank) must be evaluated in the innermost SELECT that scans the + # FTS table; the window function must live in a separate outer layer. sql = text(''' - SELECT DISTINCT ON (d.file_group_id) - d.file_group_id AS fg_id, - ds.kind AS kind, - ds.ordinal AS ordinal, - ds.label AS label, - ts_headline('english', ds.content, q, - 'MaxWords=20,MinWords=5,ShortWord=3,StartSel=[[WROLPI_HL]],StopSel=[[/WROLPI_HL]]') AS snippet - FROM doc_section ds - JOIN doc d ON d.id = ds.doc_id, - websearch_to_tsquery('english', :q) q - WHERE d.file_group_id = ANY(:ids) AND ds.tsv @@ q - ORDER BY d.file_group_id, ts_rank(ds.tsv, q) DESC, ds.ordinal ASC - ''') - rows = session.execute(sql, {'q': search_str, 'ids': list(file_group_ids)}).fetchall() + SELECT fg_id, kind, ordinal, label, snippet FROM ( + SELECT fg_id, kind, ordinal, label, snippet, + row_number() OVER (PARTITION BY fg_id + ORDER BY fts_rank ASC, ordinal ASC) AS rn + FROM ( + SELECT d.file_group_id AS fg_id, + ds.kind AS kind, + ds.ordinal AS ordinal, + ds.label AS label, + snippet(doc_section_fts, 0, '[[WROLPI_HL]]', '[[/WROLPI_HL]]', '…', 20) AS snippet, + doc_section_fts.rank AS fts_rank + FROM doc_section_fts + JOIN doc_section ds ON ds.id = doc_section_fts.rowid + JOIN doc d ON d.id = ds.doc_id + WHERE doc_section_fts MATCH :q AND d.file_group_id IN :ids + ) + ) WHERE rn = 1 + ''').bindparams(bindparam('ids', expanding=True)) + rows = session.execute(sql, {'q': match, 'ids': list(file_group_ids)}).fetchall() hints = {} for row in rows: hints[row['fg_id']] = { diff --git a/modules/docs/models.py b/modules/docs/models.py index 73432573..ff5d08ac 100644 --- a/modules/docs/models.py +++ b/modules/docs/models.py @@ -1,10 +1,10 @@ import pathlib from typing import Optional -from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, Computed, or_ -from sqlalchemy.orm import relationship, Session, deferred +from sqlalchemy import Column, Integer, BigInteger, ForeignKey, String, Text, Index, or_ +from sqlalchemy.orm import relationship, Session -from wrolpi.common import ModelHelper, Base, tsvector +from wrolpi.common import ModelHelper, Base from wrolpi.db import get_db_session from wrolpi.files.models import FileGroup @@ -139,23 +139,23 @@ class DocSection(ModelHelper, Base): For EPUB files, one row per spine item with `ordinal` as the 0-based spine index. For PDF files, one row per page with `ordinal` as the 1-based page number. - Each row maintains its own tsvector for per-section full-text search, which lets us - deep-link from a search result into a specific chapter or page. + Each row is indexed by the `doc_section_fts` FTS5 table for per-section full-text search, + which lets us deep-link from a search result into a specific chapter or page. """ __tablename__ = 'doc_section' __table_args__ = ( Index('doc_section_doc_id_idx', 'doc_id', 'ordinal'), - Index('doc_section_tsv_idx', 'tsv', postgresql_using='gin'), ) - id = Column(BigInteger, primary_key=True) + # SQLite requires exactly "INTEGER PRIMARY KEY" for the rowid alias (FTS5 content_rowid). + id = Column(BigInteger().with_variant(Integer, 'sqlite'), primary_key=True) doc_id = Column(Integer, ForeignKey('doc.id', ondelete='CASCADE'), nullable=False) kind = Column(String, nullable=False) ordinal = Column(Integer, nullable=False) label = Column(String) + # Full-text search over `content` is provided by the external-content FTS5 table + # `doc_section_fts` (see `wrolpi.fts`). content = Column(Text) - tsv = deferred(Column(tsvector, Computed( - "to_tsvector('english'::regconfig, COALESCE(content, ''))"))) doc = relationship('Doc', back_populates='sections') diff --git a/modules/inventory/migrate.py b/modules/inventory/migrate.py index 20a21a56..65e08293 100644 --- a/modules/inventory/migrate.py +++ b/modules/inventory/migrate.py @@ -70,8 +70,8 @@ def _legacy_inventories_from_db() -> list: inventories = [] try: with get_db_curs() as curs: - curs.execute("SELECT to_regclass('public.inventory')") - if curs.fetchone()[0] is None: + curs.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='inventory'") + if curs.fetchone() is None: return [] curs.execute('SELECT id, name, created_at, viewed_at FROM inventory WHERE deleted_at IS NULL') rows = curs.fetchall() @@ -79,7 +79,7 @@ def _legacy_inventories_from_db() -> list: inv = dict(id=row[0], name=row[1], created_at=row[2], viewed_at=row[3], items=[]) curs.execute( 'SELECT brand, name, item_size, unit, count, category, subcategory, expiration_date ' - 'FROM item WHERE inventory_id = %s AND deleted_at IS NULL', (row[0],)) + 'FROM item WHERE inventory_id = ? AND deleted_at IS NULL', (row[0],)) for item in curs.fetchall(): inv['items'].append(dict( brand=item[0], name=item[1], item_size=item[2], unit=item[3], count=item[4], diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py index e41904e9..fd71e210 100644 --- a/modules/videos/__init__.py +++ b/modules/videos/__init__.py @@ -1,4 +1,5 @@ import asyncio +import json from typing import Callable, List, Tuple from sqlalchemy import or_ @@ -82,19 +83,23 @@ def video_cleanup(): with get_db_curs(commit=True) as curs: # Delete all Videos if the FileModel no longer contains a video or audio file. curs.execute(''' - WITH deleted AS - (UPDATE file_group SET model = null - WHERE model = 'video' - AND mimetype NOT LIKE 'video/%%' - AND mimetype NOT LIKE 'audio/%%' - RETURNING id) - DELETE - FROM video - WHERE file_group_id = ANY (select id from deleted) + UPDATE file_group + SET model = NULL + WHERE model = 'video' + AND mimetype NOT LIKE 'video/%' + AND mimetype NOT LIKE 'audio/%' + RETURNING id ''') + deleted_ids = [i['id'] for i in curs.fetchall()] + if deleted_ids: + curs.execute(''' + DELETE + FROM video + WHERE file_group_id IN (SELECT value FROM json_each(:ids)) + ''', dict(ids=json.dumps(deleted_ids))) # Claim all Videos in a Channel's directory for that Channel. But, only if they have not yet been claimed. curs.execute(''' - UPDATE video v + UPDATE video AS v SET channel_id = c.id FROM channel c INNER JOIN collection col ON col.id = c.collection_id @@ -123,13 +128,13 @@ def claim_videos_for_channels(channel_ids: List[int]): logger.info(f'Claiming Videos for {len(channel_ids)} channel(s)') with get_db_curs(commit=True) as curs: curs.execute(''' - UPDATE video v + UPDATE video AS v SET channel_id = c.id FROM channel c INNER JOIN collection col ON col.id = c.collection_id LEFT JOIN file_group fg ON fg.directory = col.directory - OR fg.directory LIKE col.directory || '/%%' + OR fg.directory LIKE col.directory || '/%' WHERE v.channel_id IS NULL AND fg.id = v.file_group_id - AND c.id = ANY (%s) - ''', (list(channel_ids),)) + AND c.id IN (SELECT value FROM json_each(:channel_ids)) + ''', dict(channel_ids=json.dumps(list(channel_ids)))) diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py index 76a56132..738d64f5 100644 --- a/modules/videos/channel/lib.py +++ b/modules/videos/channel/lib.py @@ -2,7 +2,7 @@ import pathlib from pathlib import Path from typing import List, Union -from sqlalchemy import or_, func, desc, asc +from sqlalchemy import or_, func, desc from sqlalchemy.orm import Session, joinedload from wrolpi import flags @@ -203,6 +203,9 @@ async def search_channels_by_name(session: Session, name: str, limit: int = 5, order_by_video_count: bool = False) -> List[Channel]: name = name or '' name_no_spaces = ''.join(name.split(' ')) + # SQLite sorts by byte value; emulate the old locale-aware (en_US.UTF-8) ordering: compare + # names case-insensitively and without spaces, ties broken with lowercase names first. + name_order = (func.lower(func.replace(Collection.name, ' ', '')).asc(), Collection.name.desc()) if order_by_video_count: stmt = session.query(Channel, func.count(Video.id).label('video_count')) \ .join(Collection) \ @@ -212,7 +215,7 @@ async def search_channels_by_name(session: Session, name: str, limit: int = 5, )) \ .outerjoin(Video, Video.channel_id == Channel.id) \ .group_by(Channel.id, Collection.id, Collection.name) \ - .order_by(desc('video_count'), asc(Collection.name)) \ + .order_by(desc('video_count'), *name_order) \ .limit(limit) channels = [i[0] for i in stmt] else: @@ -222,7 +225,7 @@ async def search_channels_by_name(session: Session, name: str, limit: int = 5, Collection.name.ilike(f'%{name}%'), Collection.name.ilike(f'%{name_no_spaces}%'), )) \ - .order_by(asc(Collection.name)) \ + .order_by(*name_order) \ .limit(limit) channels = stmt.all() return channels diff --git a/modules/videos/common.py b/modules/videos/common.py index 29c43fdc..d1f4eaad 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -364,15 +364,14 @@ async def update_view_counts_and_censored(channel_id: int): with get_db_curs(commit=True) as curs: # Update the view_count for each video. stmt = ''' - WITH source AS (select * from json_to_recordset(%s::json) as (id text, view_count bigint)) UPDATE video - SET view_count = s.view_count - FROM source as s - WHERE source_id = s.id - AND channel_id = %s - RETURNING video.id AS updated_ids \ + SET view_count = CAST(json_extract(s.value, '$.view_count') AS INTEGER) + FROM json_each(:view_counts) AS s + WHERE video.source_id = json_extract(s.value, '$.id') + AND video.channel_id = :channel_id + RETURNING video.id \ ''' - curs.execute(stmt, (view_counts_str, channel_id)) + curs.execute(stmt, {'view_counts': view_counts_str, 'channel_id': channel_id}) count = len(curs.fetchall()) logger.info(f'Updated {count} view counts in DB for {channel_name}.') @@ -380,13 +379,13 @@ async def update_view_counts_and_censored(channel_id: int): with get_db_curs(commit=True) as curs: # Set FileGroup.censored if the video is no longer on the Channel. stmt = ''' - UPDATE file_group fg - SET censored = NOT (v.source_id = ANY (%(source_ids)s)) + UPDATE file_group AS fg + SET censored = (v.source_id NOT IN (SELECT value FROM json_each(:source_ids))) FROM video v WHERE v.file_group_id = fg.id - AND v.channel_id = %(channel_id)s - RETURNING fg.id, fg.censored \ + AND v.channel_id = :channel_id + RETURNING id, censored \ ''' - curs.execute(stmt, {'channel_id': channel_id, 'source_ids': source_ids}) + curs.execute(stmt, {'channel_id': channel_id, 'source_ids': json.dumps(source_ids)}) censored = len([i for i in curs.fetchall() if i['censored']]) logger.info(f'Set {censored} censored videos for {channel_name}.') diff --git a/modules/videos/lib.py b/modules/videos/lib.py index e3682aca..68e6fe47 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -1,6 +1,7 @@ import contextlib import dataclasses import html +import json import pathlib import re from dataclasses import dataclass, field @@ -988,13 +989,13 @@ async def get_statistics(): -- total videos COUNT(v.id) AS "videos", -- total videos downloaded over the past week/month/year - COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 week') AS "week", - COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 month') AS "month", - COUNT(v.id) FILTER (WHERE published_datetime >= current_date - interval '1 year') AS "year", + COUNT(v.id) FILTER (WHERE published_datetime >= datetime('now', '-7 days')) AS "week", + COUNT(v.id) FILTER (WHERE published_datetime >= datetime('now', '-1 month')) AS "month", + COUNT(v.id) FILTER (WHERE published_datetime >= datetime('now', '-1 year')) AS "year", -- sum of all video lengths in seconds - COALESCE(SUM(fg.length), 0) AS "sum_duration", + COALESCE(SUM(fg.length), 0) AS "sum_duration", -- sum of all video file sizes - COALESCE(SUM(fg.size), 0)::BIGINT AS "sum_size", + COALESCE(SUM(fg.size), 0) AS "sum_size", -- largest video COALESCE(MAX(fg.size), 0) AS "max_size", -- Videos may or may not have comments. @@ -1010,23 +1011,24 @@ async def get_statistics(): video_stats = dict(curs.fetchone()) # Get the total videos downloaded every month for the past two years. + # (Not parameterized, so `%` needs no escaping.) Like the old Postgres query, months + # without any videos are absent from the results. curs.execute(''' - SELECT DATE_TRUNC('month', months.a), - COUNT(video.id)::BIGINT, - SUM(size)::BIGINT AS "size" - FROM generate_series( - date_trunc('month', current_date) - interval '2 years', - date_trunc('month', current_date) - interval '1 month', - '1 month'::interval) AS months(a), - video + SELECT strftime('%Y-%m-01 00:00:00', fg.published_datetime) AS "month", + COUNT(video.id) AS "count", + SUM(fg.size) AS "size" + FROM video LEFT JOIN file_group fg on video.file_group_id = fg.id - WHERE published_datetime >= date_trunc('month', months.a) - AND published_datetime < date_trunc('month', months.a) + interval '1 month' - AND published_datetime IS NOT NULL + WHERE fg.published_datetime IS NOT NULL + AND fg.published_datetime >= datetime('now', 'start of month', '-24 months') + AND fg.published_datetime < datetime('now', 'start of month') GROUP BY 1 ORDER BY 1 ''') monthly_videos = [dict(i) for i in curs.fetchall()] + for monthly_video in monthly_videos: + # The frontend expects a datetime, not the TEXT that SQLite stores. + monthly_video['month'] = datetime.fromisoformat(monthly_video['month']) historical_stats = dict(monthly_videos=monthly_videos) historical_stats['average_count'] = (sum(i['count'] for i in monthly_videos) // len(monthly_videos)) \ @@ -1039,7 +1041,7 @@ async def get_statistics(): COUNT(c.id) FILTER ( WHERE col.tag_id IS NOT NULL ) AS "tagged_channels" FROM channel c LEFT JOIN collection col on col.id = c.collection_id - LEFT JOIN public.tag t on t.id = col.tag_id + LEFT JOIN tag t on t.id = col.tag_id ''') channel_stats = dict(curs.fetchone()) ret = dict(statistics=dict( @@ -1153,16 +1155,21 @@ def find_orphaned_video_files(directory: pathlib.Path) -> Generator[pathlib.Path # Query returns directory and files - files may contain relative or absolute paths # Use indexed directory column for efficient lookup + # (Not parameterized, so `%` needs no escaping.) curs.execute(f''' SELECT directory, files FROM file_group WHERE - mimetype NOT LIKE 'video%%' - AND (directory = '{directory_str}' OR directory LIKE '{directory_str}/%%') + mimetype NOT LIKE 'video%' + AND (directory = '{directory_str}' OR directory LIKE '{directory_str}/%') ''') for row in curs.fetchall(): fg_directory = pathlib.Path(row[0]) if row[0] else None - for file_info in row[1]: + files = row[1] + if isinstance(files, str): + # Raw cursor reads return the JSON column as TEXT. + files = json.loads(files) + for file_info in files: path = pathlib.Path(file_info['path']) if not path.is_absolute() and fg_directory: # Resolve relative path using directory diff --git a/modules/videos/models.py b/modules/videos/models.py index 9ac88208..573a43a9 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -2,8 +2,7 @@ import json import pathlib from typing import Optional, Dict, List -from sqlalchemy import Column, Integer, String, Boolean, Date, ForeignKey, BigInteger, Index, text -from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy import Column, Integer, String, Boolean, Date, ForeignKey, BigInteger, Index, text, JSON from sqlalchemy.orm import relationship, Session, deferred from sqlalchemy.orm.collections import InstrumentedList @@ -30,12 +29,14 @@ class Video(ModelHelper, Base): Index('video_channel_id_idx', 'channel_id'), Index('video_source_id_idx', 'source_id'), Index('video_view_count_idx', 'view_count'), + # Do not reuse ids of deleted Videos (like Postgres sequences). + {'sqlite_autoincrement': True}, ) id = Column(Integer, primary_key=True) source_id = Column(String) # The id from yt-dlp view_count = Column(BigInteger) # The view count from the ChannelDownloader (or from initial download) - ffprobe_json = deferred(Column(JSONB)) # Data that is fetched once from ffprobe (ffmpeg) + ffprobe_json = deferred(Column(JSON)) # Data that is fetched once from ffprobe (ffmpeg) have_comments = Column(Boolean, default=False) # see `get_missing_videos_comments` comments_failed = Column(Boolean, default=False) # see `get_missing_videos_comments` @@ -164,19 +165,19 @@ class Video(ModelHelper, Base): ROW_NUMBER() OVER (ORDER BY published_datetime ASC) AS row_number FROM file_group fg LEFT OUTER JOIN video v on fg.id = v.file_group_id - WHERE v.channel_id = %(channel_id)s + WHERE v.channel_id = :channel_id AND fg.published_datetime IS NOT NULL) SELECT v_id FROM numbered_videos WHERE row_number IN (SELECT row_number + i FROM numbered_videos CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n - WHERE fg_id = %(fg_id)s) \ + WHERE fg_id = :fg_id) \ ''' else: # No videos near this Video with upload dates, recommend the files next to this Video. # Only recommend videos in the same Channel (or similarly without a Channel). - channel_where = 'WHERE v.channel_id = %(channel_id)s' if self.channel_id \ + channel_where = 'WHERE v.channel_id = :channel_id' if self.channel_id \ else 'WHERE v.channel_id IS NULL' stmt = f''' WITH numbered_videos AS ( @@ -192,7 +193,7 @@ class Video(ModelHelper, Base): SELECT row_number+i FROM numbered_videos CROSS JOIN (SELECT -1 AS i UNION ALL SELECT 0 UNION ALL SELECT 1) n - WHERE fg_id = %(fg_id)s + WHERE fg_id = :fg_id ) ''' logger.debug(stmt) @@ -584,7 +585,8 @@ class Channel(ModelHelper, Base): Index('channel_source_id', 'source_id'), Index('channel_total_size_idx', 'total_size'), Index('channel_video_count_idx', 'video_count'), - Index('channel_url_key', 'url', unique=True, postgresql_where=text('url IS NOT NULL')), + Index('channel_url_key', 'url', unique=True, postgresql_where=text('url IS NOT NULL'), + sqlite_where=text('url IS NOT NULL')), ) id = Column(Integer, primary_key=True) # name and directory are stored in Collection, accessed via properties @@ -600,7 +602,7 @@ class Channel(ModelHelper, Base): total_size = Column(BigInteger, default=0, nullable=False) # update_channel_size minimum_frequency = Column(Integer) # update_channel_minimum_frequency - info_json = deferred(Column(JSONB)) + info_json = deferred(Column(JSON)) info_date = Column(Date) videos: InstrumentedList = relationship('Video', primaryjoin='Channel.id==Video.channel_id') @@ -961,10 +963,10 @@ class Channel(ModelHelper, Base): (SELECT COUNT(DISTINCT tf.file_group_id) FROM tag_file tf INNER JOIN video v ON v.file_group_id = tf.file_group_id - WHERE v.channel_id = %(id)s) AS "video_tags" + WHERE v.channel_id = :id) AS "video_tags" FROM video LEFT JOIN file_group fg on fg.id = video.file_group_id - WHERE channel_id = %(id)s \ + WHERE channel_id = :id \ ''' curs.execute(stmt, dict(id=self.id)) return dict(curs.fetchone()) @@ -976,15 +978,15 @@ class Channel(ModelHelper, Base): with get_db_curs() as curs: stmt = ''' SELECT COUNT(fg.id) AS "video_file_groups", - COUNT(fg.id) FILTER ( WHERE v.channel_id = %(id)s ) AS "assigned_to_channel", + COUNT(fg.id) FILTER ( WHERE v.channel_id = :id ) AS "assigned_to_channel", COUNT(fg.id) FILTER ( WHERE v.channel_id IS NOT NULL - AND v.channel_id != %(id)s ) AS "assigned_to_other_channels", + AND v.channel_id != :id ) AS "assigned_to_other_channels", COUNT(fg.id) FILTER ( WHERE v.id IS NULL OR v.channel_id IS NULL ) AS "unassigned" FROM file_group fg LEFT JOIN video v ON v.file_group_id = fg.id - WHERE (fg.directory = %(directory)s OR fg.directory LIKE %(directory)s || '/%%') - AND (fg.mimetype LIKE 'video/%%' OR fg.mimetype LIKE 'audio/%%') \ + WHERE (fg.directory = :directory OR fg.directory LIKE :directory || '/%') + AND (fg.mimetype LIKE 'video/%' OR fg.mimetype LIKE 'audio/%') \ ''' curs.execute(stmt, dict(id=self.id, directory=str(self.directory))) return dict(curs.fetchone()) diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py index 2a496c0b..fe71b97c 100644 --- a/modules/videos/test/test_api.py +++ b/modules/videos/test/test_api.py @@ -74,11 +74,11 @@ async def test_refresh_videos(async_client, test_session, test_directory, simple video_file = test_directory / 'foo.mp4' with get_db_curs(commit=True) as curs: stmt = "INSERT INTO file_group (mimetype, directory, primary_path, indexed, files, model)" \ - " values ('video/mp4', %(directory)s, %(primary_path)s, true, %(files)s, 'video') RETURNING id" + " values ('video/mp4', :directory, :primary_path, true, :files, 'video') RETURNING id" params = {'directory': str(test_directory), 'primary_path': str(video_file), 'files': '[]'} curs.execute(stmt, params) video4_id = curs.fetchall()[0][0] - stmt = "INSERT INTO video (file_group_id) values (%(video_id)s)" + stmt = "INSERT INTO video (file_group_id) values (:video_id)" curs.execute(stmt, {'video_id': str(video4_id)}) await async_client.post('/api/files/refresh') @@ -198,8 +198,10 @@ async def test_search_videos_file(test_session, test_directory, video_with_searc # Captions are d_text, so a deep search is required to match them. # Repeated runs should return the same result for _ in range(2): - # Only videos with a b are returned, ordered by the amount of b's - await assert_video_search(search_str='b', deep=True, assert_ids=[1, 2, 3, 4]) + # Only videos with a b are returned, ordered by their rank. (FTS5's bm25 normalizes by + # document length, so the short 'b b' caption outranks longer captions with more b's; + # Postgres' ts_rank ordered by raw frequency.) + await assert_video_search(search_str='b', deep=True, assert_ids=[3, 1, 2, 4]) # The default (fast) search does not match captions. await assert_video_search(search_str='b', assert_ids=[]) @@ -220,11 +222,11 @@ async def test_search_videos_file(test_session, test_directory, video_with_searc # only video 1 has e and d await assert_video_search(search_str='e d', deep=True, assert_ids=[1, ]) - # video 1 and 4 have b and e, but 1 has more - await assert_video_search(search_str='b e', deep=True, assert_ids=[1, 4]) + # Videos 1 and 4 have b and e; 4's caption is shorter so bm25 ranks it higher. + await assert_video_search(search_str='b e', deep=True, assert_ids=[4, 1]) # Check totals are correct even with a limit - await assert_video_search(search_str='b', deep=True, limit=2, assert_ids=[1, 2], assert_total=4) + await assert_video_search(search_str='b', deep=True, limit=2, assert_ids=[3, 1], assert_total=4) @pytest.mark.asyncio diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index 5e30f463..ac87cfec 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -8,6 +8,7 @@ from sqlalchemy import or_ from sqlalchemy.orm import Session from modules.videos.models import Video, Channel +from wrolpi import fts from wrolpi.common import logger, limit_concurrent, wrol_mode_check from wrolpi.dates import now from wrolpi.db import get_db_session @@ -58,8 +59,8 @@ VIDEO_ORDERS = { '-size': 'fg.size DESC, LOWER(fg.primary_path) DESC', 'length': 'fg.length ASC, LOWER(fg.primary_path) ASC', '-length': 'fg.length DESC, LOWER(fg.primary_path) DESC', - 'size_to_duration': 'CASE WHEN fg.length > 0 THEN fg.size::float / fg.length ELSE fg.size END ASC, LOWER(fg.primary_path) ASC', - '-size_to_duration': 'CASE WHEN fg.length > 0 THEN fg.size::float / fg.length ELSE fg.size END DESC, LOWER(fg.primary_path) DESC', + 'size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END ASC, LOWER(fg.primary_path) ASC', + '-size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END DESC, LOWER(fg.primary_path) DESC', 'viewed': 'fg.viewed ASC', '-viewed': 'fg.viewed DESC', 'view_count': 'v.view_count ASC', @@ -97,41 +98,42 @@ def search_videos( ) -> Tuple[List[dict], int]: tag_names = tag_names or [] # Search videos and audio-only files. - wheres = ["(fg.mimetype LIKE 'video/%%' OR fg.mimetype LIKE 'audio/%%')"] + wheres = ["(fg.mimetype LIKE 'video/%' OR fg.mimetype LIKE 'audio/%')"] if censored: # Only return videos which are no longer available for download. wheres.append('fg.censored = TRUE') joins = list() join_video = False - params = dict(search_str=search_str, offset=offset or 0) + # Explicit JSON nulls bypass the schema defaults; SQLite rejects LIMIT/OFFSET NULL. + limit = int(limit) if limit else 20 + params = dict(search_str=search_str, offset=int(offset) if offset else 0) if channel_id: - wheres.append('v.channel_id = %(channel_id)s') + wheres.append('v.channel_id = :channel_id') joins.append('LEFT JOIN channel c ON c.id = v.channel_id') join_video = True params['channel_id'] = channel_id - if search_str: + # `deep` searches d_text (captions) as well; the default abc search is much smaller and faster. + # The subquery join is required: snippet() cannot coexist with COUNT(*) OVER() in one SELECT. + fts_search = fts.file_group_search_join(search_str, deep=deep, headlines=bool(headline)) \ + if search_str else None + if fts_search: # A search_str was provided by the user, modify the query to filter by it. - # `textsearch` includes d_text (captions); `textsearch_abc` is much smaller and faster. - ts_col = 'textsearch' if deep else 'textsearch_abc' - select_columns = f'fg.id, ts_rank(fg.{ts_col}, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total' - wheres.append(f'fg.{ts_col} @@ websearch_to_tsquery(%(search_str)s)') - params['search_str'] = search_str + select_columns = f'fg.id, {fts_search.rank_select}, COUNT(*) OVER() AS total' + joins.append(fts_search.join) + params.update(fts_search.params) else: - # No search_str provided. Get id and total only. + # No (usable) search_str provided. Get id and total only. select_columns = 'fg.id, COUNT(*) OVER() AS total' wheres, params = tag_append_sub_select_where(wheres, params, tag_names) - if search_str and headline: - headline = ''', - ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline", - ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline", - ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline", - ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"''' + if fts_search and headline: + # The title headline is computed in Python by `handle_file_group_search_results`. + headline_selects = ', fts.b_headline, fts.c_headline, fts.d_headline' else: - headline = '' + headline_selects = '' # Convert the user-friendly order by into a real order by, restrict what can be interpolated by using the # whitelist. @@ -155,14 +157,14 @@ def search_videos( stmt = f''' SELECT {select_columns} - {headline} + {headline_selects} FROM file_group fg {join} {where} ORDER BY {order_by} - OFFSET %(offset)s LIMIT {int(limit)} + LIMIT {int(limit)} OFFSET :offset '''.strip() - logger.debug(stmt, params) + logger.debug(f'{stmt} {params}') results, total = handle_file_group_search_results(stmt, params) return results, total diff --git a/modules/zim/lib.py b/modules/zim/lib.py index b0e8b196..abf16ad8 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -1,5 +1,6 @@ import asyncio import functools +import json import pathlib import re from datetime import datetime @@ -132,7 +133,7 @@ def get_all_entries_tags(): }, """ stmt = ''' - SELECT tz.zim_id, tz.zim_entry, array_agg(t.name)::TEXT[] + SELECT tz.zim_id, tz.zim_entry, json_group_array(t.name) FROM tag t LEFT JOIN tag_zim tz on t.id = tz.tag_id GROUP BY 1, 2 \ @@ -143,6 +144,7 @@ def get_all_entries_tags(): for zim_id, zim_entry, tag_names in curs.fetchall(): if not zim_id: continue + tag_names = json.loads(tag_names) try: entries[zim_id][zim_entry] = tag_names except KeyError: @@ -195,8 +197,7 @@ def headline_zim(session: Session, search_str: str, zim_id: int, tag_names: List entries_tag_names = get_entries_tags([i.path for i in entries], zim_id) - # Always get headlines even if there is no search_str. This is because Postgres will return the start of each - # article. + # Always get headlines even if there is no search_str. Non-matches get the leading text of each article. headlines = extract_headlines(articles, search_str) search = list() diff --git a/modules/zim/models.py b/modules/zim/models.py index 797746ab..49b568c2 100644 --- a/modules/zim/models.py +++ b/modules/zim/models.py @@ -251,14 +251,16 @@ class Zim(Base, ModelHelper): def _entries_with_tags_process(limit: int, offset: int, params: dict, session: Session, tags_sub_select: str): with get_db_curs() as curs: curs.execute(tags_sub_select, params) - zim_ids_entries: List[Tuple[int, int]] = list(curs.fetchall()) + # sqlite3.Row is not a tuple; convert for the `tuple_(...).in_()` below. + zim_ids_entries: List[Tuple[int, str]] = [tuple(row) for row in curs.fetchall()] + # GROUP BY dedupes entries with multiple Tags (SQLite has no DISTINCT ON). tag_zim_entries = session.query(TagZimEntry, Zim) \ .join(Zim, Zim.id == TagZimEntry.zim_id) \ .filter(tuple_(TagZimEntry.zim_id, TagZimEntry.zim_entry).in_(zim_ids_entries)) \ + .group_by(TagZimEntry.zim_id, TagZimEntry.zim_entry) \ .order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry) \ - .offset(offset).limit(limit) \ - .distinct(TagZimEntry.zim_id, TagZimEntry.zim_entry) + .offset(offset).limit(limit) entries = list() for tag_zim_entry, zim in tag_zim_entries: diff --git a/modules/zim/test/test_api.py b/modules/zim/test/test_api.py index 6cd83808..3fe4b699 100644 --- a/modules/zim/test/test_api.py +++ b/modules/zim/test/test_api.py @@ -17,27 +17,27 @@ async def test_zim_search(async_client, test_session, test_zim, tag_factory, ass test_zim.tag_entry(test_session, tag1.name, 'two') test_session.commit() - # Test searching with only `search_str`. + # Test searching with only `search_str`. (Ranks/headlines come from FTS5 `headline_texts`.) await assert_zim_search('one', 1, { 'path': test_zim.path, 'search': [ # Entry "One" contains a 'one' in the title, and the text. - {'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271}, + {'path': 'one', 'headline': 'This is the first item', 'rank': 1e-06}, # Home only contains one 'one'. - {'path': 'home', 'rank': 0.06079271, 'headline': 'file for testing.\nOne', }, + {'path': 'home', 'rank': 8.8e-07, 'headline': '…an example ZIM file for testing.\nOne\nTwo', }, ], 'estimate': 2, }) await assert_zim_search('two', 1, { 'path': test_zim.path, - 'search': [{'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271}, - {'path': 'home', 'headline': 'testing.\nOne\nTwo', 'rank': 0.06079271}], + 'search': [{'path': 'two', 'headline': 'This is the second item', 'rank': 1e-06}, + {'path': 'home', 'headline': '…One\nTwo', 'rank': 8.8e-07}], 'estimate': 2, }) await assert_zim_search('item', 1, { 'path': test_zim.path, - 'search': [{'path': 'one', 'headline': 'first item', 'rank': 0.06079271}, - {'path': 'two', 'headline': 'second item', 'rank': 0.06079271}], + 'search': [{'path': 'one', 'headline': 'This is the first item', 'rank': 1e-06}, + {'path': 'two', 'headline': 'This is the second item', 'rank': 1e-06}], 'estimate': 2, }) @@ -45,21 +45,21 @@ async def test_zim_search(async_client, test_session, test_zim, tag_factory, ass await assert_zim_search('one', 1, tag_names=[tag2.name, ], expected={ 'search': [ # Only entry "One" is tagged with Tag 2. - {'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271}, + {'path': 'one', 'headline': 'This is the first item', 'rank': 1e-06}, ], 'estimate': 1, }) await assert_zim_search('one', 1, tag_names=[tag1.name, tag2.name], expected={ 'search': [ # Only "One" is tagged with Tag 1 and contains "one". - {'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271}, + {'path': 'one', 'headline': 'This is the first item', 'rank': 1e-06}, ], 'estimate': 1, }) await assert_zim_search('two', 1, tag_names=[tag1.name, ], expected={ 'search': [ # Only "Two" was tagged with Tag 1 and contains "two". - {'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271}, + {'path': 'two', 'headline': 'This is the second item', 'rank': 1e-06}, ], 'estimate': 1, }) diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index 47404174..b11532d1 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -172,8 +172,8 @@ async def test_zim_tags_config(async_client, test_session, test_directory, test_ assert [(i.tag.name, i.zim.path, i.zim_entry) for i in tag_zim_entries] \ == [('Tag2', test_zim.path, 'one'), ('Tag1', test_zim.path, 'two')] - # Tags were recreated. - assert {i[0] for i in test_session.query(tags.Tag.id)} == {3, 4} + # Tags were recreated. (SQLite reuses row ids after DELETE, so assert by name rather than by id.) + assert {i.name for i in test_session.query(tags.Tag)} == {'Tag1', 'Tag2'} def test_zim_all_entries(test_session, test_zim): @@ -376,7 +376,8 @@ async def test_zim_subscription_download_import(async_client, test_session): for _ in range(1): # Import twice. # Importing the config restores the Download and ZimSubscription. await import_downloads_config() - recurring, once = test_session.query(Download).order_by(Download.frequency) + # NULLS LAST because SQLite sorts NULLs first (Postgres sorted them last). + recurring, once = test_session.query(Download).order_by(Download.frequency.nullslast()) assert once.url == 'https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2021-03.zim' assert test_session.query(Download).count() == 2 assert recurring.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_' diff --git a/repair.sh b/repair.sh index 2b802f61..41f92523 100755 --- a/repair.sh +++ b/repair.sh @@ -147,24 +147,10 @@ chmod 0440 /etc/sudoers.d/90-wrolpi # Verify this new file is valid. visudo -c -f /etc/sudoers.d/90-wrolpi -# Heal Postgres cluster ownership before configuring the database. A stray -# recursive chown can leave cluster files unreadable by the server: the -# postmaster stays up but new backends die with FATAL: could not open file -# "global/pg_filenode.map": Permission denied. Same healing as -# reset_api_db.sh, but without dropping the database. -if id postgres >/dev/null 2>&1; then - [ -d /var/lib/postgresql ] && chown -R postgres:postgres /var/lib/postgresql 2>/dev/null || : - [ -d /etc/postgresql ] && chown -R postgres:postgres /etc/postgresql 2>/dev/null || : - systemctl restart postgresql 2>/dev/null || : -fi - -# Configure Postgresql. Do this after the API is stopped. Never fatal: a -# repair/upgrade must complete its system-level work even when the database -# is down (e.g. it is the reboot after this repair that revives Postgres). -/opt/wrolpi/scripts/initialize_api_db.sh || \ - echo "WARNING: could not initialize the database; the API will report if it stays down." -# wrolpi user is superuser so they can import maps. -sudo -iu postgres psql -c "alter user wrolpi with superuser" || : +# WROLPi uses SQLite inside the media directory; the API creates/migrates the database at +# startup. Never fatal: the media drive may not be mounted yet. +(cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 /opt/wrolpi/main.py db upgrade) || \ + echo "WARNING: db upgrade failed (media directory unavailable?); the API will retry at startup." # Create the media directory. This should be mounted by the maintainer. [ -d /media/wrolpi ] || mkdir /media/wrolpi diff --git a/requirements.txt b/requirements.txt index 55f0326d..f0d8950a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ jc==1.25.6 libzim==3.10.0 mock==5.2.0 psutil==7.2.2 -psycopg2-binary==2.9.12 pydantic>=2.0.0 PyMuPDF>=1.27.2.3 pytest-asyncio==1.4.0 diff --git a/scripts/dev_build.sh b/scripts/dev_build.sh index 264acdee..096fb942 100755 --- a/scripts/dev_build.sh +++ b/scripts/dev_build.sh @@ -6,6 +6,5 @@ docker volume create --name=openstreetmap-rendered-tiles git submodule update --init docker compose pull docker compose build --parallel -docker compose up -d db docker compose run --rm api db upgrade docker compose stop diff --git a/scripts/find_corrupt_videos.py b/scripts/find_corrupt_videos.py index 842a1417..b2236261 100755 --- a/scripts/find_corrupt_videos.py +++ b/scripts/find_corrupt_videos.py @@ -32,22 +32,24 @@ def get_incomplete_videos(limit: int, offset: int, channel_id: int) -> List[dict stmt = ''' WITH all_videos AS ( -- Extract 'codec_type' from all 'stream' objects of each video. - -- {'streams': [{'codec_type': 'video'}, {'codec_type': 'audio'}]} -> {video,audio} - select v.id, array_agg(s ->> 'codec_type')::TEXT[] AS codec_types - from video v - cross join lateral jsonb_array_elements(ffprobe_json -> 'streams') s - where channel_id = %(channel_id)s + -- {'streams': [{'codec_type': 'video'}, {'codec_type': 'audio'}]} -> ["video","audio"] + select v.id, json_group_array(json_extract(s.value, '$.codec_type')) AS codec_types + from video v, json_each(v.ffprobe_json, '$.streams') s + where channel_id = :channel_id group by v.id) SELECT all_videos.id AS video_id, fg.primary_path, all_videos.codec_types FROM all_videos LEFT JOIN video v ON v.id = all_videos.id - LEFT JOIN public.file_group fg on fg.id = v.file_group_id + LEFT JOIN file_group fg on fg.id = v.file_group_id -- Filter out videos that have video AND audio streams. - WHERE NOT all_videos.codec_types @> '{video,audio}'::TEXT[] - AND fg.primary_path like '/media/wrolpi/videos/%%' + WHERE NOT ( + EXISTS (SELECT 1 FROM json_each(all_videos.codec_types) WHERE value = 'video') + AND EXISTS (SELECT 1 FROM json_each(all_videos.codec_types) WHERE value = 'audio') + ) + AND fg.primary_path like '/media/wrolpi/videos/%' ORDER BY all_videos.id - LIMIT %(limit)s - OFFSET %(offset)s + LIMIT :limit + OFFSET :offset ''' curs.execute(stmt, params) return [dict(i) for i in curs.fetchall()] diff --git a/scripts/initialize_api_db.sh b/scripts/initialize_api_db.sh deleted file mode 100755 index 455b9aae..00000000 --- a/scripts/initialize_api_db.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# Creates the wrolpi database user and database if they don't exist. -# It is always safe to run this script. - -# Re-execute this script if it wasn't called with sudo. -if [ $EUID != 0 ]; then - sudo "$0" "$@" - exit $? -fi - -set -x - -# Create WROLPi API database. -sudo -iu postgres psql -c '\l' | grep wrolpi || ( - sudo -iu postgres createuser -s wrolpi # superuser so maps can be imported - sudo -iu postgres psql -c "alter user postgres password 'wrolpi'" - sudo -iu postgres psql -c "alter user wrolpi password 'wrolpi'" - sudo -iu postgres createdb -E UTF8 -O wrolpi wrolpi - echo "Created wrolpi database" -) -sudo -iu wrolpi /bin/bash -c 'cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 main.py db upgrade' diff --git a/scripts/reset_api_db.sh b/scripts/reset_api_db.sh index bfdc88d1..386571c4 100755 --- a/scripts/reset_api_db.sh +++ b/scripts/reset_api_db.sh @@ -8,7 +8,9 @@ if [ $EUID != 0 ]; then exit $? fi -if psql -l 2>/dev/null | grep wrolpi >/dev/null; then +DB_FILE=/media/wrolpi/config/wrolpi.db + +if [ -f "${DB_FILE}" ]; then yes_or_no "Are you sure you want to reset the API database? All data will be lost." || exit 0 fi @@ -16,19 +18,10 @@ set -x systemctl stop wrolpi-api -# Heal Postgres cluster ownership before touching the database. A stray recursive chown or an -# interrupted install can leave cluster files unreadable by the server; new backends then die -# with: FATAL: could not open file "global/pg_filenode.map": Permission denied -if id postgres >/dev/null 2>&1; then - [ -d /var/lib/postgresql ] && chown -R postgres:postgres /var/lib/postgresql - [ -d /etc/postgresql ] && chown -R postgres:postgres /etc/postgresql - systemctl restart postgresql -fi +# Delete the SQLite database (and its WAL sidecars), if it exists. +rm -f "${DB_FILE}" "${DB_FILE}-wal" "${DB_FILE}-shm" -# Delete the WROLPi API DB, if it exists. -sudo -iu postgres dropdb wrolpi -sudo -iu postgres dropuser wrolpi - -/bin/bash /opt/wrolpi/scripts/initialize_api_db.sh +# Recreate the database, run all migrations. +sudo -iu wrolpi /bin/bash -c 'cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 main.py db upgrade' systemctl start wrolpi-api diff --git a/scripts/reset_tags.sh b/scripts/reset_tags.sh index 2e5ac136..ef35031e 100755 --- a/scripts/reset_tags.sh +++ b/scripts/reset_tags.sh @@ -12,11 +12,13 @@ yes_or_no "Are you sure you want to delete all tags? This cannot be undone." || set -x +DB_FILE=/media/wrolpi/config/wrolpi.db + systemctl stop wrolpi-api # Delete all tag relationships and tags from the database. -sudo -iu postgres psql -d wrolpi -c "DELETE FROM tag_zim;" -sudo -iu postgres psql -d wrolpi -c "DELETE FROM tag_file;" -sudo -iu postgres psql -d wrolpi -c "DELETE FROM tag;" +sqlite3 "${DB_FILE}" "DELETE FROM tag_zim;" +sqlite3 "${DB_FILE}" "DELETE FROM tag_file;" +sqlite3 "${DB_FILE}" "DELETE FROM tag;" -systemctl start wrolpi-api \ No newline at end of file +systemctl start wrolpi-api diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index 02ee22e2..44278411 100755 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -96,11 +96,20 @@ find /opt/wrolpi/venv -name "*.pyc" -delete 2>/dev/null || : # Install any new Python requirements. /opt/wrolpi/venv/bin/pip3 install --upgrade -r /opt/wrolpi/requirements.txt -# Upgrade the WROLPi database. Never fatal: the database may be down and -# repair.sh (which runs at the end of this upgrade) may be exactly what fixes -# it; initialize_api_db.sh re-runs this migration there. +# Upgrade the WROLPi database. Never fatal: the media drive may not be mounted; +# the API creates/migrates the database itself at startup once the media directory is available. (cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 /opt/wrolpi/main.py db upgrade) || \ - echo "WARNING: db upgrade failed (database down?); it will be retried during repair." + echo "WARNING: db upgrade failed (media directory unavailable?); the API will retry at startup." + +# WROLPi now uses SQLite (inside the media directory); stop and disable PostgreSQL to free +# system resources. The old Postgres data is NOT deleted -- to get it back: +# sudo systemctl enable --now postgresql +if id postgres >/dev/null 2>&1; then + echo "Disabling PostgreSQL; WROLPi now uses SQLite. Your old Postgres data is untouched." + # Stop the umbrella and any per-cluster instances (postgresql@15-main, etc.). + systemctl stop 'postgresql*' 2>/dev/null || : + systemctl disable postgresql 2>/dev/null || : +fi # Upgrade WROLPi Help. /opt/wrolpi/scripts/install_help_service.sh || echo "Help install failed." diff --git a/wrolpi/collections/config.py b/wrolpi/collections/config.py index 4ab1101e..e758243b 100644 --- a/wrolpi/collections/config.py +++ b/wrolpi/collections/config.py @@ -85,6 +85,7 @@ class PlaylistsConfig(ConfigFile): name = data.get('name') if not name: logger.error(f'Playlist config entry has no name, skipping: {data}') + self.import_skipped += 1 continue collection = session.query(Collection).filter_by( @@ -119,6 +120,7 @@ class PlaylistsConfig(ConfigFile): else: logger.warning(f'Playlist {name!r}: file not indexed, skipping ' f'{item_data["file"]!r}') + self.import_skipped += 1 elif 'zim' in item_data: zpath = get_media_directory() / item_data['zim'] zim = session.query(Zim).filter_by(path=str(zpath)).one_or_none() @@ -128,6 +130,7 @@ class PlaylistsConfig(ConfigFile): else: logger.warning(f'Playlist {name!r}: zim not found, skipping ' f'{item_data["zim"]!r}') + self.import_skipped += 1 elif 'url' in item_data: item = CollectionItem(collection_id=collection.id, item_kind='url', url=item_data['url']) diff --git a/wrolpi/common.py b/wrolpi/common.py index 0045d1a2..eefed8f5 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -36,7 +36,6 @@ from aiohttp import ClientResponse, ClientSession from bs4 import BeautifulSoup from selenium import webdriver from sqlalchemy import types -from sqlalchemy.ext.compiler import compiles from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Session @@ -277,7 +276,6 @@ __all__ = [ 'check_media_directory', 'chunks', 'chunks_by_stem', - 'compile_tsvector', 'create_empty_config_files', 'cum_timer', 'date_range', @@ -328,7 +326,6 @@ __all__ = [ 'trim_file_name', 'truncate_generator_bytes', 'truncate_object_bytes', - 'tsvector', 'unique_by_predicate', 'url_strip_host', 'walk', @@ -382,15 +379,6 @@ class ModelHelper: raise RuntimeError(f'{self} is not in a session!') -class tsvector(types.TypeDecorator): - impl = types.UnicodeText - - -@compiles(tsvector, 'postgresql') -def compile_tsvector(element, compiler, **kw): - return 'tsvector' - - URL_CHARS = string.ascii_lowercase + string.digits @@ -542,6 +530,20 @@ class ConfigFile: from wrolpi.api_utils import api_app api_app.shared_ctx.configs_imported[self.file_name] = value + @property + def import_skipped(self) -> int: + """Count of config entries that could not be imported (e.g. their files are not yet indexed). + + While non-zero, `save()` refuses to overwrite the config file so the skipped entries are not + lost when a dump regenerates the config from the (incomplete) database.""" + from wrolpi.api_utils import api_app + return int(api_app.shared_ctx.configs_import_skipped.get(self.file_name) or 0) + + @import_skipped.setter + def import_skipped(self, value: int): + from wrolpi.api_utils import api_app + api_app.shared_ctx.configs_import_skipped[self.file_name] = int(value) + def read_config_file(self, file: pathlib.Path = None) -> dict: file = file or self.get_file() with file.open('rt') as fh: @@ -632,6 +634,11 @@ class ConfigFile: if not self.successful_import: Events.send_config_save_failed(f'Failed to save config: {rel_path}') raise RuntimeError(f'Refusing to save config because it was never successfully imported! {rel_path}') + if self.import_skipped: + message = f'Refusing to save config because {self.import_skipped} entries were skipped' \ + f' during import: {rel_path}' + Events.send_config_save_failed(message) + raise RuntimeError(message) version = self.read_config_file(file).get('version') if version and version > self.version: message = f'Refusing to overwrite newer config ({rel_path}): {version} > {self.version}' @@ -697,6 +704,8 @@ class ConfigFile: file_str = str(get_relative_to_media_directory(file)) # Caller will set to successful if it works. self.successful_import = False + # Subclasses count entries they cannot import; a fresh import starts clean. + self.import_skipped = 0 if file.is_file(): if not self.is_valid(file): raise InvalidConfig(f'Config is invalid: {file_str}') @@ -2624,10 +2633,15 @@ def get_warn_once(message: str, logger__: logging.Logger, level=logging.ERROR): def get_global_statistics(): - from wrolpi.db import get_db_curs - with get_db_curs() as curs: - curs.execute('select pg_database_size(current_database())') - db_size = curs.fetchall()[0][0] + from wrolpi.db import get_db_file + db_size = 0 + db_file = get_db_file() + if db_file.is_file(): + db_size = db_file.stat().st_size + # Include the WAL sidecar; its contents are part of the database. + wal_file = db_file.with_name(db_file.name + '-wal') + if wal_file.is_file(): + db_size += wal_file.stat().st_size return dict( db_size=db_size, @@ -2755,51 +2769,34 @@ def extract_html_text(html: str) -> str: def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, float]]: - """Use Postgres to extract Headlines and ranks of the `search_str` from the provided entries.""" - from wrolpi.db import get_db_curs + """Extract headlines (highlighted snippets) and ranks of the `search_str` from the provided entries. - source = json.dumps([{'content': i} for i in entries]) - with get_db_curs() as curs: - stmt = ''' - WITH vectored AS ( - -- Convert the "source" json to a recordset. - with source as (select * from json_to_recordset(%s::json) AS (content TEXT)) - select to_tsvector('english'::regconfig, source.content) AS vector, - source.content - from source) - SELECT ts_headline(vectored.content, websearch_to_tsquery(%s), - 'MaxFragments=10, MaxWords=8, MinWords=7'), - ts_rank(vectored.vector, websearch_to_tsquery(%s)) - FROM vectored \ - ''' - curs.execute(stmt, [source, search_str, search_str]) - headlines = [tuple(i) for i in curs.fetchall()] + Entries which do not match get the start of their text with a rank of 0.""" + from wrolpi import fts - return headlines + return fts.headline_texts(entries, search_str, tokens=8) async def search_other_estimates(tag_names: List[str]) -> dict: """Estimate other things that are Tagged.""" - from wrolpi.db import get_db_curs + from sqlalchemy import func + from wrolpi.db import get_db_session + from wrolpi.collections.models import Collection + from wrolpi.tags import Tag + from modules.videos.models import Channel if not tag_names: return dict( channel_count=0, ) - with get_db_curs() as curs: - stmt = ''' - SELECT COUNT(c.id) - FROM channel c - INNER JOIN collection col ON col.id = c.collection_id - LEFT OUTER JOIN public.tag t on t.id = col.tag_id - WHERE t.name = %(tag_name)s \ - ''' + with get_db_session() as session: # TODO handle multiple tags - params = dict(tag_name=tag_names[0]) - curs.execute(stmt, params) - - channel_count = curs.fetchone()[0] + channel_count = session.query(func.count(Channel.id)) \ + .join(Collection, Collection.id == Channel.collection_id) \ + .join(Tag, Tag.id == Collection.tag_id) \ + .filter(Tag.name == tag_names[0]) \ + .scalar() others = dict( channel_count=channel_count, diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index f5d460ad..c7c4f479 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -44,7 +44,7 @@ from wrolpi.common import logger, await_background_tasks as await_background_tas from wrolpi.common import set_test_media_directory, Base, set_test_config from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts from wrolpi.dates import set_test_now -from wrolpi.db import postgres_engine, get_db_args +from wrolpi.db import create_wrolpi_engine from wrolpi.downloader import DownloadContext, DownloadManager, DownloadResult, Download, Downloader, \ downloads_manager_config_context, download_cache_config_context from wrolpi.errors import UnrecoverableDownloadError @@ -56,34 +56,32 @@ from wrolpi.vars import PROJECT_DIR logger = logger.getChild(__name__) -def get_test_db_engine(): - suffix = str(uuid1()).replace('-', '') - db_name = f'wrolpi_testing_{suffix}' - conn = postgres_engine.connect() - conn.execute(f'DROP DATABASE IF EXISTS {db_name}') - conn.execute(f'CREATE DATABASE {db_name}') - conn.execute('commit') - conn.close() - - test_args = get_db_args(db_name) - test_engine = create_engine('postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**test_args)) - return test_engine - - def test_db() -> Tuple[Engine, Session]: - """Create a unique SQLAlchemy engine/session for a test.""" - test_engine = get_test_db_engine() - if test_engine.engine.url.database == 'wrolpi': - raise ValueError('Refusing the test on wrolpi database!') + """Create a SQLAlchemy engine/session on a SQLite DB inside the (temporary) media directory. + The DB file is at the production location (`/config/wrolpi.db`), so each test's + TemporaryDirectory provides isolation (also under pytest-xdist) and removes the DB (and its + WAL sidecars) on teardown.""" + from wrolpi.common import get_media_directory, is_tempfile + from wrolpi.schema_ddl import install_raw_ddl + + db_file = get_media_directory() / 'config' / 'wrolpi.db' + if not is_tempfile(db_file): + raise ValueError(f'Refusing to test on a non-temporary database! {db_file}') + db_file.parent.mkdir(parents=True, exist_ok=True) + + test_engine = create_wrolpi_engine(db_file) # Create all tables. No need to check if they exist because this is a test DB. Base.metadata.create_all(test_engine, checkfirst=False) + # Triggers and FTS5 tables, exactly as the alembic baseline installs them. + with test_engine.connect() as conn: + install_raw_ddl(conn) session = sessionmaker(bind=test_engine)() return test_engine, session @pytest.fixture() -def test_session() -> Generator[Session, Any, None]: +def test_session(test_directory) -> Generator[Session, Any, None]: """Pytest Fixture to get a test database session.""" test_engine, session = test_db() @@ -98,9 +96,6 @@ def test_session() -> Generator[Session, Any, None]: session.rollback() session.close() test_engine.dispose() - conn = postgres_engine.connect() - conn.execute(f'DROP DATABASE IF EXISTS {test_engine.engine.url.database}') - conn.close() @pytest.fixture(autouse=True) @@ -417,7 +412,7 @@ def production_like_sessions(test_session) -> Generator[sessionmaker, Any, None] A dedicated NullPool engine is used and disposed on exit so its connections are released before the test DB is dropped during fixture teardown. """ - engine = create_engine(test_session.get_bind().url, poolclass=NullPool) + engine = create_wrolpi_engine(test_session.get_bind().url) maker = sessionmaker(bind=engine) sessions = [] diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py index 1c166ad2..024d9472 100644 --- a/wrolpi/contexts.py +++ b/wrolpi/contexts.py @@ -80,6 +80,8 @@ def attach_shared_contexts(app: Sanic): app.shared_ctx.config_save_lock = multiprocessing.Lock() app.shared_ctx.config_update_lock = multiprocessing.Lock() app.shared_ctx.configs_imported = manager.dict() + # Count of config entries skipped during each config's import (blocks dumps until re-imported cleanly). + app.shared_ctx.configs_import_skipped = manager.dict() # Events. app.shared_ctx.events_lock = multiprocessing.Lock() @@ -143,6 +145,7 @@ def reset_shared_contexts(app: Sanic): # Configs app.shared_ctx.configs_imported.clear() + app.shared_ctx.configs_import_skipped.clear() # Switches app.shared_ctx.switches.clear() diff --git a/wrolpi/dates.py b/wrolpi/dates.py index cf7add7a..aedf3162 100644 --- a/wrolpi/dates.py +++ b/wrolpi/dates.py @@ -206,8 +206,11 @@ def months_selector_to_where(wheres: list, params: dict, months: List[int]): if not set(months).issubset(set(range(1, 13))): raise ValueError('Months must be between 1 and 12') - wheres.append('EXTRACT (MONTH FROM published_datetime) IN %(months_list)s') - params['months_list'] = tuple(months) + # sqlite3 cannot bind a tuple; use a named placeholder for each month. + placeholders = ', '.join(f':month_{idx}' for idx in range(len(months))) + for idx, month in enumerate(months): + params[f'month_{idx}'] = month + wheres.append(f"CAST(strftime('%m', published_datetime) AS INTEGER) IN ({placeholders})") return wheres, params @@ -220,14 +223,14 @@ def date_range_to_where(wheres: list, params: dict, from_year: int, to_year: int @param params: Dict which contains the parameters passed to SQLAlchemy query. """ if from_year and to_year: - wheres.append('EXTRACT (YEAR FROM published_datetime) BETWEEN %(from_year)s AND %(to_year)s') + wheres.append("CAST(strftime('%Y', published_datetime) AS INTEGER) BETWEEN :from_year AND :to_year") params['from_year'] = from_year params['to_year'] = to_year elif from_year: - wheres.append('EXTRACT (YEAR FROM published_datetime) >= %(from_year)s') + wheres.append("CAST(strftime('%Y', published_datetime) AS INTEGER) >= :from_year") params['from_year'] = from_year elif to_year: - wheres.append('EXTRACT (YEAR FROM published_datetime) <= %(to_year)s') + wheres.append("CAST(strftime('%Y', published_datetime) AS INTEGER) <= :to_year") params['to_year'] = to_year return wheres, params diff --git a/wrolpi/db.py b/wrolpi/db.py index 5add6c65..a45b3a40 100644 --- a/wrolpi/db.py +++ b/wrolpi/db.py @@ -1,70 +1,120 @@ +"""WROLPi's database: SQLite stored inside the media directory. + +The database file lives at `/config/wrolpi.db`, next to the YAML configs it +complements, so a user's entire library (files + configs + database) travels with the drive. + +Engines are created lazily (the media directory must be known first) and use NullPool with a +fresh session per use. Every connection gets the same PRAGMAs (WAL, busy_timeout, foreign keys) +via `create_wrolpi_engine` — use that factory for any engine touching a WROLPi database. +""" +import pathlib +import sqlite3 +import threading import types from contextlib import contextmanager from typing import Tuple, List, Union, Type, Generator, Any -import psycopg2 -import psycopg2.extensions import sqlalchemy import sqlalchemy.exc -from psycopg2._psycopg import cursor +from sqlalchemy import event from sqlalchemy.orm import sessionmaker, Session from sqlalchemy.pool import NullPool from wrolpi.common import logger, Base -from wrolpi.vars import DB_NAME, DB_USER, DB_PASSWORD, DB_HOST, DB_PORT, DOCKERIZED, PYTEST +from wrolpi.vars import PYTEST logger = logger.getChild(__name__) -def get_db_args(dbname: str = None): - db_args = dict( - dbname=dbname or DB_NAME, - user=DB_USER, - password=DB_PASSWORD, - host=DB_HOST, - port=DB_PORT, +def _adapt_datetime(value): + """Store datetimes from raw SQL exactly like SQLAlchemy does: naive UTC with microseconds.""" + from datetime import timezone + if value.tzinfo is not None: + value = value.astimezone(timezone.utc).replace(tzinfo=None) + return value.strftime('%Y-%m-%d %H:%M:%S.%f') + + +# Keep raw-SQL parameter formats identical to the ORM's storage formats. +import datetime as _datetime # noqa: E402 + +sqlite3.register_adapter(_datetime.datetime, _adapt_datetime) +sqlite3.register_adapter(_datetime.date, lambda value: value.isoformat()) +sqlite3.register_adapter(pathlib.PosixPath, str) +sqlite3.register_adapter(pathlib.Path, str) + + +def get_db_file() -> pathlib.Path: + """The SQLite database file: `/config/wrolpi.db`.""" + from wrolpi.common import get_media_directory + return get_media_directory() / 'config' / 'wrolpi.db' + + +def get_db_uri() -> str: + return f'sqlite:///{get_db_file()}' + + +def create_wrolpi_engine(target: Union[str, pathlib.Path]) -> sqlalchemy.engine.Engine: + """Create a SQLAlchemy engine for a WROLPi SQLite database. + + The single place every WROLPi engine is built (production, tests, alembic) so that all + connections get identical PRAGMAs and transactional behavior.""" + uri = str(target) if str(target).startswith('sqlite') else f'sqlite:///{target}' + engine = sqlalchemy.create_engine( + uri, + poolclass=NullPool, + # NullPool + executor threads mean connections may be closed on another thread. + connect_args=dict(check_same_thread=False, timeout=30), ) - if DOCKERIZED: - # Deployed in docker, use the docker database. - db_args['user'] = 'postgres' - db_args['host'] = 'db' - elif PYTEST: - # Pytest is running, but we're not in docker, use the exposed docker container port. - db_args['user'] = 'postgres' - return db_args + @event.listens_for(engine, 'connect') + def _sqlite_on_connect(dbapi_conn, _): + # Driver-level autocommit; the `begin` listener below emits BEGIN so SQLAlchemy's + # transactions still work. (pysqlite's implicit BEGIN is broken for SQLAlchemy, and + # PRAGMA journal_mode cannot run inside a transaction.) + dbapi_conn.isolation_level = None + curs = dbapi_conn.cursor() + curs.execute('PRAGMA journal_mode=WAL') + curs.execute('PRAGMA busy_timeout=30000') + curs.execute('PRAGMA synchronous=NORMAL') + curs.execute('PRAGMA foreign_keys=ON') + curs.execute('PRAGMA recursive_triggers=ON') + curs.close() + + @event.listens_for(engine, 'begin') + def _sqlite_do_begin(conn): + conn.execute('BEGIN') + + return engine -# This engine is used to modify the databases. -connect_args = dict(application_name='wrolpi_api_super') -postgres_args = get_db_args('postgres') -postgres_engine = sqlalchemy.create_engine( - 'postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**postgres_args), - execution_options={'isolation_level': 'AUTOCOMMIT'}, connect_args=connect_args) +_engine_lock = threading.Lock() +_engine: sqlalchemy.engine.Engine = None +_session_maker: sessionmaker = None -# This engine is used for all normal tasks (except testing). -db_args = get_db_args() -connect_args = dict(application_name='wrolpi_api', connect_timeout=1) -uri = 'postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**db_args) -engine = sqlalchemy.create_engine(uri, poolclass=NullPool, connect_args=connect_args) -session_maker = sessionmaker(bind=engine) -LOGGED_ARGS = False +def get_engine() -> sqlalchemy.engine.Engine: + """The lazy singleton engine for the WROLPi database. + + Created on first use (the media directory must be known by then); recreated automatically if + the media directory (and therefore the database file) changes.""" + global _engine, _session_maker + with _engine_lock: + db_file = get_db_file() + if _engine is None or _engine.url.database != str(db_file): + if _engine is not None: + _engine.dispose() + logger.info(f'Creating database engine: {db_file}') + _engine = create_wrolpi_engine(db_file) + _session_maker = sessionmaker(bind=_engine) + return _engine def _get_db_session(): """ - This function allows the database to be wrapped during testing. See: api.test.common.wrap_test_db + This function allows the database to be wrapped during testing. See: wrolpi.conftest.test_session """ - global LOGGED_ARGS - if LOGGED_ARGS is False: - # Print the DB args for troubleshooting. - LOGGED_ARGS = True - logging_args = db_args.copy() - logging_args['password'] = '***' - logger.debug(f'DB args: {logging_args}') - - session = session_maker() + engine = get_engine() + session = _session_maker() return engine, session @@ -72,8 +122,9 @@ def get_db_context() -> Tuple[sqlalchemy.engine.Engine, Session]: """ Get a DB engine and session. """ + from wrolpi.common import is_tempfile local_engine, session = _get_db_session() - if PYTEST and not local_engine.url.database.startswith('wrolpi_testing_'): + if PYTEST and not is_tempfile(local_engine.url.database or ''): raise ValueError(f'Running tests, but a test database is not being used!! {local_engine.url=}') return local_engine, session @@ -100,41 +151,38 @@ def get_db_session(commit: bool = False) -> Generator[Session, Any, None]: @contextmanager -def get_db_conn(isolation_level=psycopg2.extensions.ISOLATION_LEVEL_DEFAULT): +def get_db_curs(commit: bool = False) -> Generator[sqlite3.Cursor, Any, None]: + """ + Context manager that yields a `sqlite3.Cursor` to execute raw SQL statements. + + Rows are `sqlite3.Row` (index and name access; `dict(row)` works). SQL uses the sqlite3 + paramstyle: `?` positional, `:name` named (a statement must use only one style). + """ local_engine, session = get_db_context() if PYTEST: # During tests, use the session's connection to avoid lock issues. # The test_session is mocked to be shared, so getting a raw_connection would # create a new connection that blocks waiting for locks held by test_session. connection = session.connection().connection + curs = connection.cursor() + curs.row_factory = sqlite3.Row try: - yield connection, session + yield curs + if commit: + connection.commit() except sqlalchemy.exc.DatabaseError: session.rollback() raise # Don't rollback in finally during tests - test_session manages this else: connection = local_engine.raw_connection() - connection.set_isolation_level(isolation_level) - try: - yield connection, session - except sqlalchemy.exc.DatabaseError: - session.rollback() - raise - finally: - # Rollback only if a transaction hasn't been committed. - if session.transaction.is_active: - connection.rollback() - - -@contextmanager -def get_db_curs(commit: bool = False, isolation_level=psycopg2.extensions.ISOLATION_LEVEL_DEFAULT): - """ - Context manager that yields a DictCursor to execute raw SQL statements. - """ - with get_db_conn(isolation_level=isolation_level) as (connection, session): - curs = connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + curs = connection.cursor() + curs.row_factory = sqlite3.Row try: + if commit: + # Take the write lock up front; busy_timeout absorbs contention. (A deferred + # transaction upgrading to a write mid-way can deadlock under concurrency.) + curs.execute('BEGIN IMMEDIATE') yield curs if commit: connection.commit() @@ -143,8 +191,9 @@ def get_db_curs(commit: bool = False, isolation_level=psycopg2.extensions.ISOLAT raise finally: # Rollback only if a transaction hasn't been committed. - if session.transaction.is_active: + if connection.in_transaction: connection.rollback() + connection.close() def get_ranked_models(ranked_primary_keys: List, model: Type[Base], session: Session) -> List[Base]: @@ -156,10 +205,61 @@ def get_ranked_models(ranked_primary_keys: List, model: Type[Base], session: Ses return results -def mogrify(curs: cursor, values: Union[List, Generator]) -> str: - values = list(values) if isinstance(values, types.GeneratorType) else values.copy() - count = len(values[0]) - # If `count = 3`, then s = '(%s,%s,%s)' - s = '(' + ','.join("%s" for _ in range(count)) + ')' - result = ',\n'.join([curs.mogrify(s, i).decode() for i in values]) - return result +# SQLite's default variable limit; multi-row VALUES clauses must stay under it. +SQLITE_MAX_VARIABLES = 32_000 + + +def values_clause(rows: Union[List, Generator]) -> Tuple[str, list]: + """Build a multi-row VALUES clause and its flat parameter list. + + >>> values_clause([(1, 'a'), (2, 'b')]) + ('(?,?),(?,?)', [1, 'a', 2, 'b']) + + The caller must chunk `rows` so that `len(rows) * width` stays under SQLITE_MAX_VARIABLES.""" + rows = list(rows) if isinstance(rows, types.GeneratorType) else rows + width = len(rows[0]) + if len(rows) * width > SQLITE_MAX_VARIABLES: + raise RuntimeError(f'values_clause: too many parameters ({len(rows)} rows x {width}); chunk the rows') + row_placeholders = '(' + ','.join(['?'] * width) + ')' + sql = ','.join([row_placeholders] * len(rows)) + params = [value for row in rows for value in row] + return sql, params + + +def named_placeholders(prefix: str, values: List, params: dict) -> str: + """Add each value to `params` under a generated name, return the comma-joined placeholders. + + >>> params = dict() + >>> named_placeholders('tag_name', ['one', 'two'], params) + ':tag_name_0, :tag_name_1' + """ + names = [] + for idx, value in enumerate(values): + name = f'{prefix}_{idx}' + params[name] = value + names.append(f':{name}') + return ', '.join(names) + + +def json_each_in(param_name: str) -> str: + """An IN-clause subquery over a JSON array parameter; bind `json.dumps(list)` as the parameter. + + Immune to SQLite's variable-count limit regardless of list size.""" + return f'(SELECT value FROM json_each(:{param_name}))' + + +def parse_db_datetime(value: Union[str, None]): + """Parse a datetime TEXT value read by a raw cursor into a UTC-aware datetime. + + (The ORM's TZDateTime does this automatically; raw cursors return the stored TEXT.)""" + if not value: + return None + if isinstance(value, _datetime.datetime): + return value + try: + parsed = _datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + parsed = _datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') + return parsed.replace(tzinfo=_datetime.timezone.utc) + + diff --git a/wrolpi/db_bootstrap.py b/wrolpi/db_bootstrap.py new file mode 100644 index 00000000..22b66ac8 --- /dev/null +++ b/wrolpi/db_bootstrap.py @@ -0,0 +1,189 @@ +"""Creates and migrates the SQLite database at API startup. + +The database lives inside the media directory (`/config/wrolpi.db`), so it may belong to +a drive that was last used by a different WROLPi: + * no database → create it and run the migrations (the configs will then be imported and a + refresh rebuilds the library index); + * database older than this code → migrate it forward; + * database NEWER than this code → refuse to write; the user must upgrade this WROLPi. +""" +import contextlib +import fcntl +import sqlite3 +import sys +from pathlib import Path +from typing import Optional + +from wrolpi import flags +from wrolpi.common import logger +from wrolpi.db import get_db_file, get_db_uri +from wrolpi.vars import PROJECT_DIR, PYTEST, DOCKERIZED + +logger = logger.getChild(__name__) + +MINIMUM_SQLITE_VERSION = (3, 40, 0) + +# WAL locking is unsafe on network filesystems; refuse to run there. +NETWORK_FS_TYPES = {'nfs', 'nfs4', 'cifs', 'smb3', 'smbfs', 'fuse.sshfs', 'fuse.rclone', '9p'} + + +def _media_fs_type(path: Path) -> Optional[str]: + """The filesystem type of the mount containing `path` (Linux only).""" + if sys.platform != 'linux': + return None + try: + mounts = Path('/proc/mounts').read_text() + except OSError: + return None + best_prefix, best_type = '', None + path_str = str(path) + for line in mounts.splitlines(): + parts = line.split() + if len(parts) < 3: + continue + mount_point, fs_type = parts[1], parts[2] + if path_str == mount_point or path_str.startswith(mount_point.rstrip('/') + '/') or mount_point == '/': + if len(mount_point) > len(best_prefix): + best_prefix, best_type = mount_point, fs_type + return best_type + + +def check_sqlite_environment(db_file: Path) -> Optional[str]: + """Returns an error message when the environment cannot safely run the database, else None.""" + if sqlite3.sqlite_version_info < MINIMUM_SQLITE_VERSION: + return f'SQLite {sqlite3.sqlite_version} is too old, WROLPi requires 3.40+' + fs_type = _media_fs_type(db_file.parent if db_file.parent.exists() else db_file.parent.parent) + if fs_type in NETWORK_FS_TYPES: + return f'Refusing to use a database on a network filesystem ({fs_type}); WAL locking is unsafe there' + return None + + +def media_directory_is_unmounted_production(media_directory: Path = None) -> bool: + """True when this is a production host whose media directory should be a mount, but is not. + + While true, the database must not be created, migrated, or even considered "up": any file at + the database path is on the root filesystem, shadowing the real database on the drive. (An + ORM connection implicitly creates an empty DB file, so file-existence checks cannot be used + to detect this situation.)""" + if PYTEST or DOCKERIZED or sys.platform != 'linux': + return False + if media_directory is None: + from wrolpi.common import get_media_directory + media_directory = get_media_directory() + if not str(media_directory).startswith('/media/'): + # A custom MEDIA_DIRECTORY (dev boxes) may legitimately not be a mount. + return False + try: + return not media_directory.is_mount() + except OSError: + return True + + +def _alembic_config(): + from alembic.config import Config + config = Config(str(PROJECT_DIR / 'alembic.ini')) + config.set_main_option('sqlalchemy.url', get_db_uri()) + return config + + +def compare_db_version() -> str: + """Compare the database's alembic version to this code's. + + Returns 'missing' (no/uninitialized DB file), 'current', 'behind' (needs migrations), or + 'ahead' (the DB was written by a newer WROLPi).""" + from alembic.script import ScriptDirectory + + db_file = get_db_file() + if not db_file.is_file(): + return 'missing' + try: + # Read-only URI probe: a plain connect would create an empty database file. + with contextlib.closing(sqlite3.connect(f'file:{db_file}?mode=ro', uri=True)) as conn: + row = conn.execute('SELECT version_num FROM alembic_version').fetchone() + current = row[0] if row else None + except sqlite3.OperationalError: + current = None + if not current: + # The file exists but was never migrated; treat like a fresh database. + return 'missing' + + script = ScriptDirectory.from_config(_alembic_config()) + if current == script.get_current_head(): + return 'current' + try: + script.get_revision(current) + return 'behind' + except Exception: + # This code does not know the DB's revision: the DB is from a newer WROLPi. + return 'ahead' + + +def ensure_db() -> bool: + """Create/migrate the database. Returns True when the database is usable at head. + + Never creates the database beneath an unmounted media directory on a production host + (that would silently start an empty library on the root filesystem).""" + from alembic import command + from wrolpi.common import get_media_directory + from wrolpi.events import Events + + db_file = get_db_file() + + if error := check_sqlite_environment(db_file): + logger.critical(error) + with contextlib.suppress(Exception): + Events.send_config_import_failed(error) + return False + + media_directory = get_media_directory() + if not media_directory.is_dir(): + logger.warning(f'Cannot create/migrate database; media directory is missing: {media_directory}') + return False + if media_directory_is_unmounted_production(media_directory): + # A production media directory that is not mounted: any database created/migrated here + # would land on the root filesystem and shadow the real database once the drive mounts. + # (Unconditional: a worker's ORM connection may have already created an empty DB file.) + logger.warning(f'Refusing to create/migrate database beneath unmounted media directory: {media_directory}') + return False + + state = compare_db_version() + if state == 'ahead': + flags.db_version_mismatch.set() + message = 'The database was written by a NEWER WROLPi (drive from another system?). ' \ + 'Upgrade this WROLPi before using this drive.' + logger.critical(message) + with contextlib.suppress(Exception): + Events.send_config_import_failed(message) + return False + + flags.db_version_mismatch.clear() + + if state in ('missing', 'behind'): + db_file.parent.mkdir(parents=True, exist_ok=True) + logger.warning(f'Migrating database ({state}): {db_file}') + command.upgrade(_alembic_config(), 'head') + + return True + + +@contextlib.contextmanager +def bootstrap_lock(): + """A non-blocking cross-process lock; yields True when this process holds the lock. + + Multiple Sanic workers may try to bootstrap simultaneously; only one needs to.""" + lock_file = get_db_file().with_suffix('.db.lock') + try: + lock_file.parent.mkdir(parents=True, exist_ok=True) + with lock_file.open('w') as fh: + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + yield False + return + try: + yield True + finally: + fcntl.flock(fh, fcntl.LOCK_UN) + except OSError: + # Cannot create the lock file (media directory missing/read-only); let the caller try. + yield True diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 1c3130fb..4243ea4e 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1,5 +1,6 @@ import asyncio import contextlib +import json import logging import multiprocessing import os @@ -23,10 +24,8 @@ from zoneinfo import ZoneInfo import feedparser import pytz from feedparser import FeedParserDict -from sqlalchemy import Column, Integer, String, Text, ForeignKey, ARRAY, Index -from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy import Column, Integer, String, Text, ForeignKey, Index, JSON from sqlalchemy.orm import Session, relationship -from sqlalchemy.sql import Delete from wrolpi import flags from wrolpi.api_utils import api_app, perpetual_signal @@ -252,14 +251,14 @@ class Download(ModelHelper, Base): # noqa sub_downloader = Column(Text) # The downloader any returned downloads should be sent. error = Column(Text) # traceback from an error during downloading. frequency = Column(Integer) # seconds between re-downloading. - info_json = Column(JSONB) # information retrieved WHILE downloading (from yt-dlp) + info_json = Column(JSON) # information retrieved WHILE downloading (from yt-dlp) last_download_attempt = Column(TZDateTime) # last time this Download was dispatched (success or failure) last_successful_download = Column(TZDateTime) location = Column(Text) # Relative App URL where the downloaded item can be viewed. next_download = Column(TZDateTime) - settings = Column(JSONB) # information about how the download should happen (video_resolution, etc.) + settings = Column(JSON) # information about how the download should happen (video_resolution, etc.) status = Column(String, default=DownloadStatus.new) # `DownloadStatus` enum. - tag_names = Column(ARRAY(Text)) + tag_names = Column(JSON) # a list of Tag names. # A Download may be associated with a Collection (downloads for channels, domains, etc.). collection_id = Column(Integer, ForeignKey('collection.id', ondelete='SET NULL')) @@ -917,13 +916,11 @@ class DownloadManager: # Ensure all Tags exist before creating downloads. if tag_names: - with get_db_curs() as curs: - stmt = 'SELECT name FROM tag WHERE name = ANY(%(tag_names)s)' - curs.execute(stmt, dict(tag_names=tag_names)) - existing_tag_names = {i[0] for i in curs.fetchall()} - missing_tag_names = set(tag_names) - existing_tag_names - if missing_tag_names: - raise ValidationError(f'Tag does not exist: {missing_tag_names.pop()}') + from wrolpi.tags import Tag + existing_tag_names = {name for name, in session.query(Tag.name).filter(Tag.name.in_(tag_names))} + missing_tag_names = set(tag_names) - existing_tag_names + if missing_tag_names: + raise ValidationError(f'Tag does not exist: {missing_tag_names.pop()}') downloads = [] # Throws an error if no downloader is found. @@ -1201,12 +1198,11 @@ class DownloadManager: @param reset_attempts: Will set `download.attempts` to 0 if True. """ - with get_db_curs(commit=True) as curs: - if reset_attempts: - stmt = "UPDATE download SET status='new', attempts=0 WHERE status='pending' OR status='deferred' OR status='failed'" - else: - stmt = "UPDATE download SET status='new' WHERE status='pending' OR status='deferred' OR status='failed'" - curs.execute(stmt) + values = dict(status=DownloadStatus.new, attempts=0) if reset_attempts else dict(status=DownloadStatus.new) + with get_db_session(commit=True) as session: + session.query(Download) \ + .filter(Download.status.in_((DownloadStatus.pending, DownloadStatus.deferred, DownloadStatus.failed))) \ + .update(values, synchronize_session=False) def get_new_downloads(self, session: Session) -> Generator[Download, None, None]: """ @@ -1345,6 +1341,19 @@ class DownloadManager: WHEN (status = 'complete') THEN 4 END''' + @staticmethod + def _parse_fe_download_row(download: dict) -> dict: + """Raw cursor rows return JSON columns as text and datetimes as naive strings; restore + the Python types the frontend serialization expects.""" + from wrolpi.db import parse_db_datetime + for key in ('settings', 'tag_names'): + if isinstance(download.get(key), str): + download[key] = json.loads(download[key]) + for key in ('last_successful_download', 'next_download'): + if download.get(key): + download[key] = parse_db_datetime(download[key]) + return download + def get_fe_downloads(self): """Get downloads for the Frontend. Uses raw SQL for faster result.""" # Use custom SQL because SQLAlchemy is slow. @@ -1369,11 +1378,11 @@ class DownloadManager: WHERE frequency IS NOT NULL ORDER BY {self._status_order}, - next_download, + next_download NULLS LAST, frequency ''' curs.execute(stmt) - recurring_downloads = list(map(dict, curs.fetchall())) + recurring_downloads = [self._parse_fe_download_row(dict(i)) for i in curs.fetchall()] stmt = f''' SELECT @@ -1393,12 +1402,12 @@ class DownloadManager: WHERE frequency IS NULL ORDER BY {self._status_order}, - last_successful_download DESC, + last_successful_download DESC NULLS FIRST, id LIMIT 100 ''' curs.execute(stmt) - once_downloads = list(map(dict, curs.fetchall())) + once_downloads = [self._parse_fe_download_row(dict(i)) for i in curs.fetchall()] stmt = ''' SELECT COUNT(id) FILTER ( @@ -1429,17 +1438,16 @@ class DownloadManager: """ Get a summary of what Downloads are happening as well as the status of the DownloadManager. """ - with get_db_curs() as curs: - stmt = """ - SELECT COUNT(*) FILTER (WHERE status = 'pending') AS pending_downloads, - COUNT(*) FILTER (WHERE frequency IS NOT NULL) AS recurring_downloads - FROM download \ - """ - curs.execute(stmt) - counts = list(map(dict, curs.fetchall()))[0] + from sqlalchemy import func + + with get_db_session() as session: + pending_downloads, recurring_downloads = session.query( + func.count(Download.id).filter(Download.status == DownloadStatus.pending), + func.count(Download.id).filter(Download.frequency != None), # noqa + ).one() summary = dict( - pending=counts['pending_downloads'], - recurring=counts['recurring_downloads'], + pending=pending_downloads, + recurring=recurring_downloads, disabled=self.is_disabled, stopped=self.is_stopped, outside_download_window=self.outside_download_window, @@ -1527,25 +1535,33 @@ class DownloadManager: return [i for i, j in zip(zagger, range(len(downloads)))][index] @staticmethod - def _delete_downloads_q(once: bool = False, status: str = None, returning=Download.id) -> Delete: - stmt = Download.__table__.delete().returning(returning) + def _delete_downloads(session: Session, once: bool = False, status: str = None, returning=Download.id) -> List: + """Delete matching Downloads; returns the requested column of the deleted rows. + + (SQLAlchemy 1.3's SQLite dialect does not support DELETE ... RETURNING, so the values are + selected before deleting, in the same transaction.)""" + query = session.query(returning) + delete = Download.__table__.delete() if once: - stmt = stmt.where(Download.frequency == None) + query = query.filter(Download.frequency == None) # noqa + delete = delete.where(Download.frequency == None) # noqa if status: - stmt = stmt.where(Download.status == status) - return stmt + query = query.filter(Download.status == status) + delete = delete.where(Download.status == status) + values = [i for i, in query.all()] + session.execute(delete) + return values def delete_completed(self, session: Session) -> List[int]: """Delete any completed download records.""" - stmt = self._delete_downloads_q(once=True, status=DownloadStatus.complete) - deleted_ids = [i for i, in session.execute(stmt).fetchall()] + deleted_ids = self._delete_downloads(session, once=True, status=DownloadStatus.complete) session.commit() return deleted_ids def delete_failed(self, session: Session): """Delete any failed download records.""" - stmt = self._delete_downloads_q(once=True, status=DownloadStatus.failed, returning=Download.url) - deleted_urls = [i for i, in session.execute(stmt).fetchall()] + deleted_urls = self._delete_downloads(session, once=True, status=DownloadStatus.failed, + returning=Download.url) # Add all downloads to permanent skip list. self.add_to_skip_list(*deleted_urls) @@ -1554,8 +1570,7 @@ class DownloadManager: def delete_once(self, session: Session): """Delete all once-download records.""" - stmt = self._delete_downloads_q(once=True) - deleted_ids = [i for i, in session.execute(stmt).fetchall()] + deleted_ids = self._delete_downloads(session, once=True) session.commit() save_downloads_config.activate_switch() return deleted_ids @@ -1564,8 +1579,8 @@ class DownloadManager: """Delete specific downloads by their IDs.""" if not download_ids: return [] - stmt = Download.__table__.delete().where(Download.id.in_(download_ids)).returning(Download.id) - deleted_ids = [i for i, in session.execute(stmt).fetchall()] + deleted_ids = [i for i, in session.query(Download.id).filter(Download.id.in_(download_ids)).all()] + session.execute(Download.__table__.delete().where(Download.id.in_(deleted_ids))) session.commit() save_downloads_config.activate_switch() return deleted_ids @@ -1588,14 +1603,13 @@ class DownloadManager: if not download_ids: return [] from sqlalchemy import and_ - stmt = Download.__table__.delete().where( - and_( - Download.id.in_(download_ids), - Download.status == DownloadStatus.complete, - Download.frequency == None # noqa - only once-downloads - ) - ).returning(Download.id) - deleted_ids = [i for i, in session.execute(stmt).fetchall()] + conditions = and_( + Download.id.in_(download_ids), + Download.status == DownloadStatus.complete, + Download.frequency == None # noqa - only once-downloads + ) + deleted_ids = [i for i, in session.query(Download.id).filter(conditions).all()] + session.execute(Download.__table__.delete().where(conditions)) session.commit() return deleted_ids diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index b2fa0c12..769e1cf2 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -18,7 +18,6 @@ from pathlib import Path from typing import Callable, List, Tuple, Union, Dict, Generator, Iterable, Set import cachetools.func -import psycopg2 from sqlalchemy import asc, or_ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -31,7 +30,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, \ get_wrolpi_config, \ unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where -from wrolpi.db import get_db_session, get_db_curs, mogrify +from wrolpi.db import get_db_session, get_db_curs, values_clause from wrolpi.downloader import download_manager, Download from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged, \ NoPrimaryFile, InvalidDirectory, IgnoredDirectoryError, UnsupportedArchive, InvalidArchiveMember @@ -685,7 +684,6 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime, progress_callback: Optional callback(processed, total) called after each chunk is processed It is assumed all files exist.""" - idempotency = str(idempotency) total_files = len(files) processed_count = 0 @@ -698,42 +696,54 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime, for group in grouped: # The primary file is the video/SingleFile/epub, etc. try: - primary_path = get_primary_file(group) - # Multiple files in this group. - primary_path: pathlib.Path = primary_path - # The primary mimetype allows modelers to find its file_groups. - mimetype = get_mimetype(primary_path) - # The group uses a common modification_datetime so the group will be re-indexed when any of it's files - # are modified. - modification_datetime = from_timestamp(max(i.stat().st_mtime for i in group)) - size = sum(i.stat().st_size for i in group) - files = json.dumps(_paths_to_files_dict(group)) - suffix = (split_path_stem_and_suffix(primary_path)[1] or '').lower() or None - values[primary_path] = (modification_datetime, mimetype, size, files, suffix) - non_primary_files = {i for i in group if i != primary_path} - except NoPrimaryFile: - # Cannot find primary path, create a `file_group` for each file. - for primary_path in group: - primary_path: pathlib.Path + try: + primary_path = get_primary_file(group) + # Multiple files in this group. + primary_path: pathlib.Path = primary_path # The primary mimetype allows modelers to find its file_groups. mimetype = get_mimetype(primary_path) - modification_datetime = from_timestamp(primary_path.stat().st_mtime) - size = primary_path.stat().st_size - files = json.dumps(_paths_to_files_dict([primary_path])) + # The group uses a common modification_datetime so the group will be re-indexed when any of it's + # files are modified. + modification_datetime = from_timestamp(max(i.stat().st_mtime for i in group)) + size = sum(i.stat().st_size for i in group) + files = json.dumps(_paths_to_files_dict(group)) suffix = (split_path_stem_and_suffix(primary_path)[1] or '').lower() or None values[primary_path] = (modification_datetime, mimetype, size, files, suffix) + non_primary_files = {i for i in group if i != primary_path} + except NoPrimaryFile: + # Cannot find primary path, create a `file_group` for each file. + for primary_path in group: + primary_path: pathlib.Path + # The primary mimetype allows modelers to find its file_groups. + mimetype = get_mimetype(primary_path) + modification_datetime = from_timestamp(primary_path.stat().st_mtime) + size = primary_path.stat().st_size + files = json.dumps(_paths_to_files_dict([primary_path])) + suffix = (split_path_stem_and_suffix(primary_path)[1] or '').lower() or None + values[primary_path] = (modification_datetime, mimetype, size, files, suffix) + except FileNotFoundError as e: + # A file was deleted between discovery and upsert (temp files, WAL sidecars, etc.). + refresh_logger.warning(f'File vanished during refresh, skipping group near {group[0]}', exc_info=e) + continue values = [(str(primary_path), str(primary_path.parent), # directory False, # `indexed` is false to force indexing by default. idempotency, *i) for primary_path, i in values.items()] with get_db_curs(commit=True) as curs: - values = mogrify(curs, values) + # SQLite's RETURNING only exposes the post-upsert row; find the previously-indexed + # paths first so re-indexing invalidations can be counted. + primary_paths = [i[0] for i in values] + curs.execute('SELECT primary_path FROM file_group WHERE indexed = true AND primary_path IN ' + '(SELECT value FROM json_each(?))', (json.dumps(primary_paths),)) + previously_indexed = {i['primary_path'] for i in curs.fetchall()} + + placeholders, params = values_clause(values) stmt = f''' INSERT INTO file_group (primary_path, directory, indexed, idempotency, modification_datetime, mimetype, size, files, suffix) - VALUES {values} + VALUES {placeholders} ON CONFLICT (primary_path) DO UPDATE SET directory=EXCLUDED.directory, @@ -749,7 +759,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime, WHEN file_group.modification_datetime = EXCLUDED.modification_datetime AND file_group.size = EXCLUDED.size - AND jsonb_array_length(file_group.files) = jsonb_array_length(EXCLUDED.files) + AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files) THEN file_group.files ELSE EXCLUDED.files END @@ -759,14 +769,15 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime, file_group.indexed = true AND file_group.modification_datetime = EXCLUDED.modification_datetime AND file_group.size = EXCLUDED.size - AND jsonb_array_length(file_group.files) = jsonb_array_length(EXCLUDED.files) + AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files) ) - RETURNING id, file_group.indexed AS old_indexed, indexed + RETURNING id, primary_path, indexed ''' - curs.execute(stmt) + curs.execute(stmt, params) results = list(curs.fetchall()) # Count the files that used to be indexed, but need to be re-indexed. - invalidated_files = len([i['id'] for i in results if i['old_indexed'] and not i['indexed']]) + invalidated_files = len([i['id'] for i in results + if i['primary_path'] in previously_indexed and not i['indexed']]) if invalidated_files: refresh_logger.info(f'Invalidated indexes of {invalidated_files} file groups near {chunk[0]}') else: @@ -775,8 +786,8 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime, if non_primary_files: # New files may have been added which change what primary paths exist. Delete any file_groups which # use paths which are not primary. - curs.execute('DELETE FROM file_group WHERE primary_path = ANY(%s)', - (list(map(str, non_primary_files)),)) + curs.execute('DELETE FROM file_group WHERE primary_path IN (SELECT value FROM json_each(?))', + (json.dumps(list(map(str, non_primary_files))),)) # Report progress after each chunk processed_count += len(chunk) @@ -786,12 +797,17 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime, def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]: """Return a new list which does not contain any file paths that are in ignored directories.""" + from wrolpi.db import get_db_file + ignored_directories = list(map(str, get_wrolpi_config().ignored_directories)) for idx, ignored_directory in enumerate(ignored_directories): ignored_directory = pathlib.Path(ignored_directory) if not ignored_directory.is_absolute(): ignored_directories[idx] = str(get_media_directory() / ignored_directory) files = [i for i in files if not any(str(i).startswith(j) for j in ignored_directories)] + # Never index the database (or its WAL sidecars), even if the user un-ignores `config`. + db_file = str(get_db_file()) + files = [i for i in files if not str(i).startswith(db_file)] return files @@ -859,32 +875,37 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim refresh_logger.info('Finished discovering files. Will now remove deleted files from DB...') with get_db_curs(commit=True) as curs: + params = [] + deleted_files = '' + if deleted: + # Delete any paths that do not exist. + deleted_files = ' OR '.join(['primary_path = ?'] * len(deleted)) + params.extend(str(i) for i in deleted) + wheres = '' if paths: # Use indexed directory column to delete any children of directories that are deleted. # Match exact directory or subdirectories conditions = [] + where_params = [] for p in paths: p_str = str(p) - conditions.append(curs.mogrify('(directory = %s OR directory LIKE %s)', (p_str, f'{p_str}/%')).decode()) - wheres = ' OR '.join(conditions) - deleted_files = '' - if deleted: - # Delete any paths that do not exist. - deleted_files = ' OR '.join([curs.mogrify('primary_path = %s', (str(i),)).decode() for i in deleted]) + conditions.append('(directory = ? OR directory LIKE ?)') + where_params.extend([p_str, f'{p_str}/%']) + wheres = ' ( (idempotency != ? OR idempotency is null) AND (' + ' OR '.join(conditions) + '))' + params.append(idempotency) + params.extend(where_params) - idempotency = curs.mogrify('%s', (idempotency,)).decode() - wheres = f' ( (idempotency != {idempotency} OR idempotency is null) AND ({wheres}))' if wheres else '' stmt = f''' DELETE FROM file_group WHERE -- Delete all known-deleted files. {deleted_files} -- Delete any files in the refreshed paths that were not updated. - {" OR " + wheres if deleted_files else wheres} + {" OR " + wheres if deleted_files and wheres else wheres} ''' refresh_logger.debug(stmt) - curs.execute(stmt) + curs.execute(stmt, params) async def apply_indexers(progress_callback: Callable[[int, int], None] = None): @@ -978,7 +999,8 @@ async def apply_indexers(progress_callback: Callable[[int, int], None] = None): refresh_logger.error(f'Failed to fix UnicodeEncodeError, skipping batch: {e2}') # Mark as indexed in a separate transaction so we don't loop forever with get_db_curs(commit=True) as curs: - curs.execute('UPDATE file_group SET indexed = TRUE WHERE id = ANY(%s)', (fg_ids,)) + curs.execute('UPDATE file_group SET indexed = TRUE WHERE id IN ' + '(SELECT value FROM json_each(?))', (json.dumps(list(fg_ids)),)) continue if file_group: @@ -1009,18 +1031,18 @@ def upsert_directories(parent_directories, directories): # Insert any directories that were created, update any directories which previously existed. with get_db_curs(commit=True) as curs: values = [(str(i.absolute()), i.name, idempotency) for i in directories] - values = mogrify(curs, values) + placeholders, params = values_clause(values) stmt = f''' - INSERT INTO directory (path, name, idempotency) VALUES {values} + INSERT INTO directory (path, name, idempotency) VALUES {placeholders} ON CONFLICT (path) DO UPDATE SET idempotency = EXCLUDED.idempotency ''' - curs.execute(stmt) + curs.execute(stmt, params) with get_db_curs(commit=True) as curs: # Delete the children of any parent directory which no longer exists. for directory in parent_directories: - stmt = f'DELETE FROM directory WHERE path LIKE %(directory)s AND idempotency != %(idempotency)s' + stmt = f'DELETE FROM directory WHERE path LIKE :directory AND idempotency != :idempotency' curs.execute(stmt, dict(directory=f'{directory.absolute()}%', idempotency=idempotency)) @@ -1063,48 +1085,60 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] @param path: Filter by primary_path using case-insensitive partial match (ILIKE). @param deep: Search all text including captions/body (d_text); slower but more thorough. """ - params = dict(offset=offset, limit=limit, search_str=search_str, url_search_str=f'%{search_str}%') + from wrolpi import fts + + # The frontend may send explicit JSON nulls (bypassing the schema defaults); SQLite rejects + # `LIMIT NULL`/`OFFSET NULL` (Postgres treated them as no-ops). + limit = int(limit) if limit else 20 + offset = int(offset) if offset else 0 + params = dict(offset=offset, limit=limit, search_str=search_str) wheres = [] selects = [] order_by = '1 ASC' joins = [] - if search_str: - # `textsearch` includes d_text (captions/body); `textsearch_abc` is much smaller and faster. - ts_col = 'textsearch' if deep else 'textsearch_abc' - wheres.append(f'{ts_col} @@ websearch_to_tsquery(%(search_str)s)') - selects.append(f'ts_rank({ts_col}, websearch_to_tsquery(%(search_str)s))') + # The fast (non-deep) path excludes d_text (captions/body) from matching. + fts_search = fts.file_group_search_join(search_str, deep=deep, headlines=bool(headline)) \ + if search_str else None + if search_str and not fts_search: + # Nothing usable to search (punctuation-only or all-negated query); match nothing, like + # an empty websearch_to_tsquery did. + return [], 0 + if fts_search: + joins.append(fts_search.join) + selects.append(fts_search.rank_select) + params.update(fts_search.params) order_by = '2 DESC' wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes) if model: params['model'] = model - wheres.append('model = %(model)s') + wheres.append('model = :model') wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag) wheres, params = months_selector_to_where(wheres, params, months) wheres, params = date_range_to_where(wheres, params, from_year, to_year) if url: + # SQLite LIKE is case-insensitive (for ASCII), matching the old ILIKE behavior. params['url_filter'] = f'%{url}%' - wheres.append('fg.url ILIKE %(url_filter)s') + wheres.append('fg.url LIKE :url_filter') if suffix: # Suffix is stored lowercased with a leading dot; normalize the caller's value to match the index. normalized = suffix if suffix.startswith('.') else f'.{suffix}' params['suffix'] = normalized.lower() - wheres.append('fg.suffix = %(suffix)s') + wheres.append('fg.suffix = :suffix') if path: params['path_filter'] = f'%{path}%' - wheres.append('fg.primary_path ILIKE %(path_filter)s') + wheres.append('fg.primary_path LIKE :path_filter') - if search_str and headline: - headline = ''', - ts_headline(fg.title, websearch_to_tsquery(%(search_str)s)) AS "title_headline", - ts_headline(fg.b_text, websearch_to_tsquery(%(search_str)s)) AS "b_headline", - ts_headline(fg.c_text, websearch_to_tsquery(%(search_str)s)) AS "c_headline", - ts_headline(fg.d_text, websearch_to_tsquery(%(search_str)s)) AS "d_headline"''' + if fts_search and headline: + # b/c/d headlines come from FTS5 snippets (computed in the fts subquery); the title + # headline is computed in Python by `handle_file_group_search_results` (title is not an + # FTS column). + headline = ', fts.b_headline, fts.c_headline, fts.d_headline' else: headline = '' @@ -1127,7 +1161,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] {join} {f"WHERE {wheres}" if wheres else ""} ORDER BY {order_by} - OFFSET %(offset)s LIMIT %(limit)s + LIMIT :limit OFFSET :offset ''' logger.debug(stmt) @@ -1145,24 +1179,19 @@ def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List """ with get_db_curs() as curs: curs.execute(statement, params) - try: - results = [dict(i) for i in curs.fetchall()] - except psycopg2.ProgrammingError: - # No videos - return [], 0 + results = [dict(i) for i in curs.fetchall()] total = results[0]['total'] if results else 0 ordered_ids = [i['id'] for i in results] - try: - extras = [dict( - ts_rank=i.get('ts_rank'), - title_headline=i.get('title_headline'), - b_headline=i.get('b_headline'), - c_headline=i.get('c_headline'), - d_headline=i.get('d_headline'), - ) for i in results] - except KeyError: - # No `ts_rank`, probably not searching `file_group.textsearch`. - extras = [] + extras = [dict( + ts_rank=i.get('ts_rank'), + # Snippet columns exist only when headlines were requested; `title_headline` is + # computed below (title is not an FTS5 column). + headlines=('b_headline' in i), + title_headline=i.get('title_headline'), + b_headline=i.get('b_headline'), + c_headline=i.get('c_headline'), + d_headline=i.get('d_headline'), + ) for i in results] with get_db_session() as session: from modules.archive.models import Archive @@ -1190,9 +1219,19 @@ def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List results[-1]['c_headline'] = extra['c_headline'] results[-1]['d_headline'] = extra['d_headline'] + search_str = params.get('search_str') if isinstance(params, dict) else None + + # When headlines were requested, compute the title headlines in Python (the FTS5 snippet + # function only works on indexed columns; `title` is not one). + if search_str and any(extra and extra.get('headlines') for extra in extras): + from wrolpi import fts + titles = [result.get('title') or '' for result in results] + for result, (headline_text, rank) in zip(results, fts.headline_texts(titles, search_str)): + if rank > 0: + result['title_headline'] = headline_text + # When searching by text, attach a `section_hint` to any doc-modeled result so # the UI can deep-link into the matching EPUB chapter or PDF page. - search_str = params.get('search_str') if isinstance(params, dict) else None if search_str: from modules.docs.lib import _fetch_section_hints # `r['id']` is the FileGroup primary key; _fetch_section_hints filters @@ -1290,7 +1329,7 @@ def get_file_statistics(): curs.execute(''' SELECT -- All items in file_group.files are real individual files. - SUM(jsonb_array_length(files)) AS "total_count", + SUM(json_array_length(files)) AS "total_count", COUNT(id) FILTER (WHERE file_group.mimetype = 'application/pdf') AS "pdf_count", COUNT(id) FILTER (WHERE file_group.mimetype = 'application/zip') AS "zip_count", COUNT(id) FILTER (WHERE file_group.mimetype LIKE 'video/%') AS "video_count", @@ -1301,7 +1340,7 @@ def get_file_statistics(): (SELECT COUNT(DISTINCT tag_file.file_group_id) FROM tag_file) AS "tagged_files", (SELECT COUNT(DISTINCT tag_zim.zim_entry) FROM tag_zim) AS "tagged_zims", (SELECT COUNT(*) FROM tag) AS "tags_count", - SUM(size)::BIGINT AS "total_size", + SUM(size) AS "total_size", (SELECT COUNT(*) FROM archive) AS archive_count FROM file_group ''') @@ -1425,10 +1464,10 @@ def get_refresh_progress() -> RefreshProgress: stmt = ''' SELECT -- Sum all the files in each FileGroup. - SUM(jsonb_array_length(files)) FILTER (WHERE idempotency = %(idempotency)s) AS "total_file_groups", - COUNT(id) FILTER (WHERE indexed IS TRUE AND idempotency = %(idempotency)s) AS "indexed", - COUNT(id) FILTER (WHERE indexed IS FALSE AND idempotency = %(idempotency)s) AS "unindexed", - COUNT(id) FILTER (WHERE model IS NOT NULL AND idempotency = %(idempotency)s) AS "modeled" + SUM(json_array_length(files)) FILTER (WHERE idempotency = :idempotency) AS "total_file_groups", + COUNT(id) FILTER (WHERE indexed IS TRUE AND idempotency = :idempotency) AS "indexed", + COUNT(id) FILTER (WHERE indexed IS FALSE AND idempotency = :idempotency) AS "unindexed", + COUNT(id) FILTER (WHERE model IS NOT NULL AND idempotency = :idempotency) AS "modeled" FROM file_group \ ''' else: @@ -1436,7 +1475,7 @@ def get_refresh_progress() -> RefreshProgress: stmt = ''' SELECT -- Sum all the files in each FileGroup. - SUM(jsonb_array_length(files)) AS "total_file_groups", + SUM(json_array_length(files)) AS "total_file_groups", COUNT(id) FILTER (WHERE indexed IS TRUE) AS "indexed", COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed", COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled" @@ -1474,7 +1513,7 @@ def mimetypes_to_sql_wheres(wheres: List[str], params: dict, mimetypes: List[str params[key] = f'{mimetype}/%' else: params[key] = f'{mimetype}%' - local_wheres.append(f'mimetype LIKE %({key})s') + local_wheres.append(f'mimetype LIKE :{key}') wheres.append(f'({" OR ".join(local_wheres)})') return wheres, params @@ -1485,24 +1524,26 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi """ Return FileGroup counts of what will be returned if the search is actually performed. - Returns both the fast (`textsearch_abc`) and deep (`textsearch`) counts so the UI can show the count + Returns both the fast (a/b/c columns) and deep (all columns) counts so the UI can show the count matching the current search mode, and offer deep search when it has more results. Both are cheap - index-bound COUNTs (no ts_rank). + index-bound COUNTs (no ranking). """ + from wrolpi import fts + wheres = [] joins = list() - params = dict(search_str=search_str, tag_names=tag_names, mimetypes=mimetypes) - group_by = '' - having = '' + params = dict() - if search_str: - # `textsearch` matches are a superset of `textsearch_abc`, so filter by the deep column and count - # the fast subset with FILTER in the same pass. - wheres.append('fg.textsearch @@ websearch_to_tsquery(%(search_str)s)') - selects = ('COUNT(*) FILTER (WHERE fg.textsearch_abc @@ websearch_to_tsquery(%(search_str)s)) AS estimate,' - ' COUNT(*) AS deep_estimate') - else: - selects = 'COUNT(*) AS estimate, COUNT(*) AS deep_estimate' + # The deep match is a superset of the fast (column-filtered) match; both are counted with the + # same statement, differing only in the MATCH expression. + deep_match = fts.translate_websearch(search_str) if search_str else None + fast_match = fts.translate_websearch(search_str, fts.ABC_COLUMNS) if search_str else None + if search_str and deep_match is None: + # Nothing usable to search; match nothing (like an empty websearch_to_tsquery). + return dict(file_groups=0, file_groups_deep=0) + if deep_match: + joins.append('JOIN file_group_fts fts ON fts.rowid = fg.id') + wheres.append('fts.file_group_fts MATCH :fts_match') wheres, params = tag_append_sub_select_where(wheres, params, tag_names, any_tag) wheres, params = mimetypes_to_sql_wheres(wheres, params, mimetypes) @@ -1512,21 +1553,23 @@ async def search_file_suggestion_count(search_str: str, tag_names: List[str], mi joins = "\n".join(joins) wheres = 'WHERE ' + "\nAND ".join(wheres) if wheres else '' stmt = f''' - SELECT {selects} + SELECT COUNT(*) AS estimate FROM file_group fg {joins} {wheres} - {group_by} - {having} ''' - logger.debug(stmt, params) + logger.debug(f'search_file_suggestion_count: {stmt} {params}') with get_db_curs() as curs: - curs.execute(stmt, params) - result = curs.fetchone() - if result: - return dict(file_groups=int(result['estimate']), file_groups_deep=int(result['deep_estimate'])) - return dict(file_groups=0, file_groups_deep=0) + if deep_match: + curs.execute(stmt, {**params, 'fts_match': deep_match}) + deep_estimate = int(curs.fetchone()['estimate']) + curs.execute(stmt, {**params, 'fts_match': fast_match}) + estimate = int(curs.fetchone()['estimate']) + else: + curs.execute(stmt, params) + estimate = deep_estimate = int(curs.fetchone()['estimate']) + return dict(file_groups=estimate, file_groups_deep=deep_estimate) MOVE_CHUNK_SIZE = 100 @@ -1571,25 +1614,23 @@ def _bulk_update_file_groups_db(session: Session, chunk_plan: Dict[pathlib.Path, if not chunk_plan: return - # Use psycopg2 cursor for safe SQL building via mogrify with get_db_curs(commit=True) as curs: - # Build VALUES list for the update: (old_path, new_dir, new_path) - values = [ - (str(old_path), str(new_path.parent), str(new_path)) + # Build the update rows: [old_path, new_dir, new_path] + updates = json.dumps([ + [str(old_path), str(new_path.parent), str(new_path)] for old_path, new_path in chunk_plan.items() - ] - values_sql = mogrify(curs, values) + ]) - # Use UPDATE FROM VALUES pattern for efficient batch update - sql = f''' + # Use UPDATE FROM json_each pattern for efficient batch update + sql = ''' UPDATE file_group AS fg SET - directory = v.new_dir, - primary_path = v.new_path, + directory = json_extract(v.value, '$[1]'), + primary_path = json_extract(v.value, '$[2]'), indexed = CASE WHEN fg.data IS NULL THEN FALSE ELSE fg.indexed END - FROM (VALUES {values_sql}) AS v(old_path, new_dir, new_path) - WHERE fg.primary_path = v.old_path + FROM json_each(?) AS v + WHERE fg.primary_path = json_extract(v.value, '$[0]') ''' - curs.execute(sql) + curs.execute(sql, (updates,)) def _json_serial(obj): @@ -1617,11 +1658,11 @@ def _bulk_update_file_groups_reorganize(updates: List[dict]): # Don't change indexed - reorganization only moves files, content unchanged curs.execute(''' UPDATE file_group - SET directory = %s, - primary_path = %s, - files = %s, - data = %s - WHERE id = %s + SET directory = ?, + primary_path = ?, + files = ?, + data = ? + WHERE id = ? ''', ( update['directory'], update['primary_path'], @@ -1654,9 +1695,8 @@ def delete_directory(directory: pathlib.Path, recursive: bool = False, force: bo else: directory.rmdir() # Always clean up DB record even if directory was already deleted - with get_db_curs(commit=True) as curs: - stmt = 'DELETE FROM directory WHERE path=%s' - curs.execute(stmt, (str(directory),)) + with get_db_session(commit=True) as session: + session.query(Directory).filter_by(path=str(directory)).delete(synchronize_session=False) async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path: @@ -2045,13 +2085,13 @@ def _bulk_insert_tag_files(tag_files: List[Tuple[int, int]]): for tag_id, file_group_id in chunk] with get_db_curs(commit=True) as curs: - values_str = mogrify(curs, values) + placeholders, params = values_clause(values) stmt = f''' INSERT INTO tag_file (tag_id, file_group_id, created_at) - VALUES {values_str} + VALUES {placeholders} ON CONFLICT (tag_id, file_group_id) DO NOTHING ''' - curs.execute(stmt) + curs.execute(stmt, params) logger.debug(f'Bulk inserted {len(chunk)} TagFiles') @@ -2070,12 +2110,12 @@ def _bulk_delete_tag_files(tag_files: List[Tuple[int, int]]): pairs = [(tag_id, file_group_id) for tag_id, file_group_id in chunk] with get_db_curs(commit=True) as curs: - pairs_str = mogrify(curs, pairs) + placeholders, params = values_clause(pairs) stmt = f''' DELETE FROM tag_file - WHERE (tag_id, file_group_id) IN ({pairs_str}) + WHERE (tag_id, file_group_id) IN (VALUES {placeholders}) ''' - curs.execute(stmt) + curs.execute(stmt, params) logger.debug(f'Bulk deleted {curs.rowcount} TagFiles') diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index f48f4c34..1d37f436 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -6,12 +6,11 @@ import urllib.parse from datetime import datetime from typing import List, Type, Optional, Iterable -from sqlalchemy import Column, String, Computed, BigInteger, Boolean, event, Index +from sqlalchemy import Column, String, BigInteger, Boolean, event, Index, Integer, JSON from sqlalchemy import types -from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import deferred, relationship, Session -from wrolpi.common import Base, ModelHelper, tsvector, logger, recursive_map, get_media_directory, \ +from wrolpi.common import Base, ModelHelper, logger, recursive_map, get_media_directory, \ get_relative_to_media_directory, unique_by_predicate, replace_file from wrolpi.dates import TZDateTime, now, from_timestamp, strptime_ms, strftime from wrolpi.db import get_db_session @@ -50,7 +49,7 @@ class FancyJSON(types.TypeDecorator): Converts datetime to ISO strings when moving into DB, and vice versa. """ - impl = JSONB + impl = JSON def process_bind_param(self, value, dialect): if value: @@ -79,12 +78,11 @@ class FileGroup(ModelHelper, Base): Index('file_group_published_datetime_idx', 'published_datetime'), Index('file_group_published_modified_datetime_idx', 'published_modified_datetime'), Index('file_group_size_ix', 'size'), - Index('file_group_textsearch_abc_idx', 'textsearch_abc'), - Index('file_group_textsearch_idx', 'textsearch'), Index('file_group_url_idx', 'url'), Index('file_group_viewed_idx', 'viewed'), ) - id: int = Column(BigInteger, primary_key=True) + # SQLite requires exactly "INTEGER PRIMARY KEY" for the rowid alias (FTS5 content_rowid). + id: int = Column(BigInteger().with_variant(Integer, 'sqlite'), primary_key=True) # Directory containing all files in this group (absolute path). # All paths in `files` and `data` are relative to this directory. @@ -116,24 +114,12 @@ class FileGroup(ModelHelper, Base): tag_files: Iterable[TagFile] = relationship('TagFile', cascade='all') + # Full-text search over these columns is provided by the external-content FTS5 table + # `file_group_fts` (see `wrolpi.fts`): a=title, b=author-ish, c=description, d=body/captions. a_text = deferred(Column(String)) b_text = deferred(Column(String)) c_text = deferred(Column(String)) d_text = deferred(Column(String)) - textsearch = deferred( - Column(tsvector, Computed(''' - setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(d_text, '')), 'D'::"char") - '''))) - # Excludes d_text (captions/body); much smaller than `textsearch`, used for the default fast search. - textsearch_abc = deferred( - Column(tsvector, Computed(''' - setweight(to_tsvector('english'::regconfig, COALESCE(a_text, '')), 'A'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(b_text, '')), 'B'::"char") || - setweight(to_tsvector('english'::regconfig, COALESCE(c_text, '')), 'C'::"char") - '''))) def __repr__(self): m = f'model={self.model}' if self.model else f'mimetype={self.mimetype}' diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index ae999fe6..122858fb 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -286,6 +286,51 @@ async def test_files_search_recent(test_session, test_directory, async_client, v assert [i['name'] for i in response.json['file_groups']] == ['foo.mp4'] +@pytest.mark.asyncio +async def test_files_search_null_params(test_session, test_directory, async_client, video_file_factory, + refresh_files): + """Explicit JSON nulls (as the dashboard sends) must not break the search. + + Regression test: the frontend sends `{"search_str": null, "offset": null, ...}`; explicit + nulls bypass the schema defaults and reached `OFFSET NULL`, which Postgres treated as 0 but + SQLite rejects with 'datatype mismatch'.""" + video_file_factory(test_directory / 'foo.mp4') + await refresh_files() + file_group = test_session.query(FileGroup).one() + file_group.set_viewed(datetime(2000, 1, 1, 1, 1, 1, tzinfo=timezone.utc)) + test_session.commit() + + # The exact dashboard "recent files" payload. + body = {'search_str': None, 'offset': None, 'limit': 12, 'any_tag': False, 'tag_names': [], 'order': 'viewed'} + request, response = await async_client.post('/api/files/search', json=body) + assert response.status_code == HTTPStatus.OK, response.json + assert [i['name'] for i in response.json['file_groups']] == ['foo.mp4'] + + # Null limit as well. + body = {'search_str': None, 'offset': None, 'limit': None} + request, response = await async_client.post('/api/files/search', json=body) + assert response.status_code == HTTPStatus.OK, response.json + assert response.json['file_groups'] + + +@pytest.mark.asyncio +async def test_files_search_stemming(test_session, async_client, make_files_structure, refresh_files, + assert_files_search): + """Searching any grammatical variant of a word finds files named with other variants. + + End-to-end stemming contract: "watch" must find "watching", "fishes" must find "fishing".""" + make_files_structure(['watching birds.mp4', 'fishing guide.txt', 'two cats.txt', 'unrelated.txt']) + await refresh_files() + + await assert_files_search('watch', [dict(name='watching birds.mp4')]) + await assert_files_search('watched', [dict(name='watching birds.mp4')]) + await assert_files_search('fish', [dict(name='fishing guide.txt')]) + await assert_files_search('fishes', [dict(name='fishing guide.txt')]) + await assert_files_search('cat', [dict(name='two cats.txt')]) + # Stemmers do not lemmatize: derived words are their own terms (same as the old engine). + await assert_files_search('runner', []) + + @pytest.mark.asyncio async def test_files_search(test_session, async_client, make_files_structure, assert_files_search): # You can search an empty directory. diff --git a/wrolpi/files/test/test_worker.py b/wrolpi/files/test/test_worker.py index cbbd0c60..c1b8856d 100644 --- a/wrolpi/files/test/test_worker.py +++ b/wrolpi/files/test/test_worker.py @@ -348,9 +348,9 @@ async def test_compare_file_groups_cleans_up_work_table_on_aborted_tx( async for p in original(root): yield p test_session.commit() - # Poison the transaction so any later DDL fails unless rolled back. + # A failed statement mid-transaction must not prevent the work-table cleanup. try: - test_session.execute(text('SELECT FROM does_not_exist_xyz')) + test_session.execute(text('SELECT * FROM does_not_exist_xyz')) except Exception: pass raise RuntimeError('simulated crash mid-scan') @@ -360,8 +360,8 @@ async def test_compare_file_groups_cleans_up_work_table_on_aborted_tx( await compare_file_groups(test_directory) test_session.rollback() - leftover = [row.tablename for row in test_session.execute(text( - "SELECT tablename FROM pg_tables WHERE tablename LIKE 'fs_files_%'" + leftover = [row[0] for row in test_session.execute(text( + "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'fs_files_%'" ))] assert leftover == [], f'orphaned work tables: {leftover}' diff --git a/wrolpi/files/worker.py b/wrolpi/files/worker.py index c4510217..445f7eee 100644 --- a/wrolpi/files/worker.py +++ b/wrolpi/files/worker.py @@ -5,6 +5,7 @@ This module provides memory-efficient comparison of potentially millions of file on resource-constrained devices like Raspberry Pi. """ import asyncio +import json import os import pathlib import shlex @@ -14,6 +15,7 @@ import time import uuid from collections import OrderedDict, defaultdict from dataclasses import dataclass, field +from datetime import datetime, timezone from enum import Enum, auto from multiprocessing.queues import Queue from pathlib import Path @@ -234,7 +236,8 @@ async def find_directories(directories: list[Path]) -> set[Path]: find_args = [str(d) for d in directories] find_args.extend(['-type', 'd', '-not', '-path', '*/.*']) for ignored in _get_normalized_ignored_directories(): - find_args.extend(['-not', '-path', f'{ignored}/*']) + # Exclude the ignored directory itself as well as its contents. + find_args.extend(['-not', '-path', ignored, '-not', '-path', f'{ignored}/*']) proc = await asyncio.create_subprocess_exec( 'find', *find_args, @@ -265,6 +268,9 @@ async def _stream_filesystem_paths(root: Path) -> AsyncGenerator[str, None]: find_args = [str(root), '-type', 'f', '-not', '-path', '*/.*'] for ignored in _get_normalized_ignored_directories(): find_args.extend(['-not', '-path', f'{ignored}/*']) + # Never index the database (or its WAL sidecars), even if `config` is un-ignored. + from wrolpi.db import get_db_file + find_args.extend(['-not', '-path', f'{get_db_file()}*']) proc = await asyncio.create_subprocess_exec( 'find', *find_args, @@ -283,26 +289,31 @@ async def _stream_filesystem_paths(root: Path) -> AsyncGenerator[str, None]: await proc.wait() -def _insert_fs_batch(session, table_name: str, batch: list): - """Bulk insert filesystem files using unnest for speed.""" +def _datetime_text_to_epoch(value) -> float: + """Convert a raw DB datetime (TEXT `YYYY-MM-DD HH:MM:SS.ffffff`, UTC) to a Unix epoch. + + Preserves microsecond precision (SQL `unixepoch()` truncates to whole seconds, which + would break sub-second mtime comparisons).""" + if not value: + return 0 + dt = value if isinstance(value, datetime) else datetime.fromisoformat(value) + if dt.tzinfo is None: + # TZDateTime stores all datetimes as naive UTC text. + dt = dt.replace(tzinfo=timezone.utc) + return dt.timestamp() + + +def _insert_fs_batch(conn, table_name: str, batch: list): + """Bulk insert filesystem files using executemany for speed.""" if not batch: return - session.execute( + conn.execute( text(f""" INSERT INTO {table_name} (directory, filename, stem) - SELECT * - FROM unnest( - CAST(:dirs AS text[]), - CAST(:files AS text[]), - CAST(:stems AS text[]) - ) + VALUES (:directory, :filename, :stem) ON CONFLICT DO NOTHING """), - { - "dirs": [b[0] for b in batch], - "files": [b[1] for b in batch], - "stems": [b[2] for b in batch], - } + [{"directory": b[0], "filename": b[1], "stem": b[2]} for b in batch], ) @@ -314,7 +325,7 @@ async def compare_file_groups( """ Compare FileGroups in DB to filesystem. Memory efficient for Raspberry Pi. - Uses a uniquely-named PostgreSQL work table to handle millions of files without + Uses a uniquely-named SQLite work table to handle millions of files without loading everything into Python memory. Returns which FileGroups are new, deleted, modified, or unchanged. @@ -323,15 +334,17 @@ async def compare_file_groups( the subprocess scanning the filesystem. """ root = root or get_media_directory() - # Per-call table name so concurrent scans don't collide, and so commits on the - # session (which can come from unrelated callers in tests, where get_db_session - # is mocked to share one session) don't drop the table mid-scan. + # Per-call table name so concurrent scans don't collide. The work table is a TEMP table on + # its own dedicated connection: it never shares a transaction (or locks) with the session, + # it is invisible to other connections, and it is dropped automatically when the connection + # closes — orphaned work tables are impossible. table_name = f'fs_files_{uuid.uuid4().hex[:12]}' with get_db_session() as session: + work_conn = session.get_bind().connect() try: - session.execute(text(f""" - CREATE TABLE {table_name} + work_conn.execute(text(f""" + CREATE TEMP TABLE {table_name} ( directory TEXT NOT NULL, filename TEXT NOT NULL, @@ -353,7 +366,7 @@ async def compare_file_groups( batch.append((directory, filename, stem)) if len(batch) >= batch_size: - _insert_fs_batch(session, table_name, batch) + _insert_fs_batch(work_conn, table_name, batch) file_count += len(batch) batch = [] if progress_callback: @@ -363,24 +376,25 @@ async def compare_file_groups( await path_gen.aclose() # Ensure subprocess is killed on cancellation if batch: - _insert_fs_batch(session, table_name, batch) + _insert_fs_batch(work_conn, table_name, batch) file_count += len(batch) logger.info(f'Scanned {file_count} files from filesystem') # Index for fast grouping - session.execute(text(f"CREATE INDEX ON {table_name}(directory, stem)")) - session.execute(text(f"ANALYZE {table_name}")) + work_conn.execute( + text(f"CREATE INDEX IF NOT EXISTS {table_name}_dir_stem_idx ON {table_name}(directory, stem)")) + work_conn.execute(text(f"ANALYZE {table_name}")) # Get filesystem files grouped by (directory, stem) fs_groups: Dict[Tuple[str, str], Set[str]] = {} - result = session.execute(text(f""" - SELECT directory, stem, array_agg(filename) as files + result = work_conn.execute(text(f""" + SELECT directory, stem, json_group_array(filename) as files FROM {table_name} GROUP BY directory, stem """)) for row in result: - fs_groups[(row.directory, row.stem)] = set(row.files) + fs_groups[(row.directory, row.stem)] = set(json.loads(row.files)) logger.info(f'Found {len(fs_groups)} file groups on filesystem') @@ -394,31 +408,37 @@ async def compare_file_groups( SELECT fg.id, fg.directory, fg.primary_path, - fg.files IS NULL OR fg.files = '[]'::jsonb as is_empty, - EXTRACT(EPOCH FROM fg.modification_datetime) as mtime_epoch, - CASE - WHEN fg.files IS NOT NULL AND jsonb_typeof(fg.files) = 'array' AND - fg.files != '[]'::jsonb - THEN (SELECT array_agg(f ->> 'path') - FROM jsonb_array_elements(fg.files) as f) - ELSE NULL - END as files + (fg.files IS NULL + OR (json_type(fg.files) = 'array' + AND json_array_length(fg.files) = 0)) as is_empty, + fg.modification_datetime as modification_datetime, + fg.files as files FROM file_group fg WHERE fg.directory IS NOT NULL AND (fg.directory = :root OR fg.directory LIKE :root_pattern) """), {"root": root_str, "root_pattern": f"{root_str}/%"}) for row in result: + # Raw TEXT datetime (UTC, microsecond precision); unixepoch() would truncate to + # whole seconds which breaks the sub-second MTIME_TOLERANCE_SECONDS comparison. + mtime_epoch = _datetime_text_to_epoch(row.modification_datetime) if row.is_empty: empty_count += 1 stem, _ = split_path_stem_and_suffix(Path(row.primary_path)) key = (row.directory, stem) - db_groups[key].append((row.id, set(), row.mtime_epoch or 0)) - elif row.files: - first_file = row.files[0] + db_groups[key].append((row.id, set(), mtime_epoch)) + else: + raw_files = row.files + if isinstance(raw_files, str): + raw_files = json.loads(raw_files) + if not isinstance(raw_files, list) or not raw_files: + # Matches the old Postgres behavior: non-array `files` was skipped. + continue + files = [f['path'] for f in raw_files] + first_file = files[0] stem, _ = split_path_stem_and_suffix(Path(row.directory) / first_file) key = (row.directory, stem) - db_groups[key].append((row.id, set(row.files), row.mtime_epoch or 0)) + db_groups[key].append((row.id, set(files), mtime_epoch)) logger.info(f'Found {len(db_groups)} file groups in database') if empty_count: @@ -486,23 +506,11 @@ async def compare_file_groups( modified=modified, ) finally: - # Drop the work table regardless of success/failure. Prefer the caller's - # session so the DROP joins the same transaction as the CREATE (cleanup - # follows whatever the caller does with the tx). If the tx is aborted — - # e.g. an external commit + later SQL error left us mid-error — Postgres - # rejects DDL on the session; fall back to a fresh connection so the - # already-committed work table doesn't orphan. + # Closing the dedicated connection drops its TEMP work table. try: - session.execute(text(f"DROP TABLE IF EXISTS {table_name}")) + work_conn.close() except Exception: - from wrolpi.db import get_db_context - try: - engine, _ = get_db_context() - with engine.connect() as conn: - with conn.begin(): - conn.execute(text(f"DROP TABLE IF EXISTS {table_name}")) - except Exception: - logger.exception(f'Failed to drop work table {table_name}') + logger.exception(f'Failed to close work table connection for {table_name}') async def build_move_plan_bulk( @@ -558,196 +566,191 @@ async def build_move_plan_bulk( progress_callback(i, total_sources * 2) # *2 because file collection is second half with get_db_session() as session: - # Create temp table for source paths - session.execute(text(""" - CREATE TEMP TABLE IF NOT EXISTS move_sources - ( - source_path TEXT PRIMARY KEY, - source_type TEXT NOT NULL, - source_directory TEXT - ) ON COMMIT DROP - """)) - session.execute(text("TRUNCATE move_sources")) + try: + # Create temp table for source paths. SQLite has no ON COMMIT DROP; the table is + # explicitly dropped below on success/failure (and IF NOT EXISTS + DELETE guard + # against any leftover from a previous failed run on the same connection). + session.execute(text(""" + CREATE TEMP TABLE IF NOT EXISTS move_sources + ( + source_path TEXT PRIMARY KEY, + source_type TEXT NOT NULL, + source_directory TEXT + ) + """)) + session.execute(text("DELETE FROM move_sources")) - # Collect all paths to insert: file paths + expanded directory contents - all_file_paths: list[pathlib.Path] = [] - files_collected = 0 - # Use total_sources as the denominator for the second half of progress - total_items = len(file_sources) + len(dir_sources) + # Collect all paths to insert: file paths + expanded directory contents + all_file_paths: list[pathlib.Path] = [] + files_collected = 0 + # Use total_sources as the denominator for the second half of progress + total_items = len(file_sources) + len(dir_sources) - # For files, collect the file and its shared-stem siblings - for i, source in enumerate(file_sources, 1): - files = glob_shared_stem(source) - all_file_paths.extend(files) - files_collected += 1 - # Report progress (second half: 50-100%) - if progress_callback and files_collected % 50 == 0: - progress = total_sources + (files_collected * total_sources // max(total_items, 1)) - progress_callback(progress, total_sources * 2) + # For files, collect the file and its shared-stem siblings + for i, source in enumerate(file_sources, 1): + files = glob_shared_stem(source) + all_file_paths.extend(files) + files_collected += 1 + # Report progress (second half: 50-100%) + if progress_callback and files_collected % 50 == 0: + progress = total_sources + (files_collected * total_sources // max(total_items, 1)) + progress_callback(progress, total_sources * 2) - # For directories, walk and collect all files - dir_file_mapping: Dict[pathlib.Path, pathlib.Path] = {} # file -> source_dir - for i, source_dir in enumerate(dir_sources, 1): - for f in walk(source_dir): - if f.is_file(): - all_file_paths.append(f) - dir_file_mapping[f] = source_dir - files_collected += 1 - # Report progress during directory walking (every 10 directories) - if progress_callback and files_collected % 10 == 0: - progress = total_sources + (files_collected * total_sources // max(total_items, 1)) - progress_callback(progress, total_sources * 2) + # For directories, walk and collect all files + dir_file_mapping: Dict[pathlib.Path, pathlib.Path] = {} # file -> source_dir + for i, source_dir in enumerate(dir_sources, 1): + for f in walk(source_dir): + if f.is_file(): + all_file_paths.append(f) + dir_file_mapping[f] = source_dir + files_collected += 1 + # Report progress during directory walking (every 10 directories) + if progress_callback and files_collected % 10 == 0: + progress = total_sources + (files_collected * total_sources // max(total_items, 1)) + progress_callback(progress, total_sources * 2) - # Final progress update before deduplication - if progress_callback: - progress_callback(total_sources * 2, total_sources * 2) + # Final progress update before deduplication + if progress_callback: + progress_callback(total_sources * 2, total_sources * 2) - # Deduplicate - all_file_paths = list(set(all_file_paths)) + # Deduplicate + all_file_paths = list(set(all_file_paths)) - # Bulk insert file paths into temp table - if all_file_paths: - paths_data = [(str(p), 'file', str(p.parent)) for p in all_file_paths] - session.execute( - text(""" - INSERT INTO move_sources (source_path, source_type, source_directory) - SELECT * - FROM unnest( - CAST(:paths AS text[]), - CAST(:types AS text[]), - CAST(:dirs AS text[]) - ) - ON CONFLICT DO NOTHING - """), - { - "paths": [p[0] for p in paths_data], - "types": [p[1] for p in paths_data], - "dirs": [p[2] for p in paths_data], - } - ) + # Bulk insert file paths into temp table + if all_file_paths: + session.execute( + text(""" + INSERT INTO move_sources (source_path, source_type, source_directory) + VALUES (:path, :type, :dir) + ON CONFLICT DO NOTHING + """), + [{"path": str(p), "type": 'file', "dir": str(p.parent)} for p in all_file_paths], + ) - # Also insert deleted sources (for pre-moved files lookup) - if deleted_sources: - deleted_data = [(str(p), 'deleted', str(p.parent)) for p in deleted_sources] - session.execute( - text(""" - INSERT INTO move_sources (source_path, source_type, source_directory) - SELECT * - FROM unnest( - CAST(:paths AS text[]), - CAST(:types AS text[]), - CAST(:dirs AS text[]) - ) - ON CONFLICT DO NOTHING - """), - { - "paths": [p[0] for p in deleted_data], - "types": [p[1] for p in deleted_data], - "dirs": [p[2] for p in deleted_data], - } - ) + # Also insert deleted sources (for pre-moved files lookup) + if deleted_sources: + session.execute( + text(""" + INSERT INTO move_sources (source_path, source_type, source_directory) + VALUES (:path, :type, :dir) + ON CONFLICT DO NOTHING + """), + [{"path": str(p), "type": 'deleted', "dir": str(p.parent)} for p in deleted_sources], + ) - # Single bulk query to get all FileGroups matching our sources - result = session.execute(text(""" - SELECT fg.id, fg.primary_path, fg.directory, ms.source_path, ms.source_type - FROM file_group fg - JOIN move_sources ms ON fg.primary_path = ms.source_path - """)) + # Single bulk query to get all FileGroups matching our sources + result = session.execute(text(""" + SELECT fg.id, fg.primary_path, fg.directory, ms.source_path, ms.source_type + FROM file_group fg + JOIN move_sources ms ON fg.primary_path = ms.source_path + """)) - # Build lookup of found FileGroups - fg_by_path: Dict[str, Tuple[int, str, str]] = {} # path -> (id, primary_path, directory) - for row in result: - fg_by_path[row.source_path] = (row.id, row.primary_path, row.directory) + # Build lookup of found FileGroups + fg_by_path: Dict[str, Tuple[int, str, str]] = {} # path -> (id, primary_path, directory) + for row in result: + fg_by_path[row.source_path] = (row.id, row.primary_path, row.directory) - logger.info(f'build_move_plan_bulk: found {len(fg_by_path)} FileGroups for {len(all_file_paths)} files') + logger.info(f'build_move_plan_bulk: found {len(fg_by_path)} FileGroups for {len(all_file_paths)} files') - # Build plan for file sources (direct files, not from directories) - file_source_set = set() - for source in file_sources: - files = glob_shared_stem(source) - file_source_set.update(str(f) for f in files) + # Build plan for file sources (direct files, not from directories) + file_source_set = set() + for source in file_sources: + files = glob_shared_stem(source) + file_source_set.update(str(f) for f in files) - # Process files from direct file sources - for path_str, (fg_id, primary_path, directory) in fg_by_path.items(): - path = pathlib.Path(path_str) - primary_path = pathlib.Path(primary_path) + # Process files from direct file sources + for path_str, (fg_id, primary_path, directory) in fg_by_path.items(): + path = pathlib.Path(path_str) + primary_path = pathlib.Path(primary_path) - # Is this from a file source or a directory source? - if path_str in file_source_set: - # Direct file source -> destination / filename - new_path = destination / primary_path.name - elif path in dir_file_mapping: - # From directory source -> destination / source_dir.name / relative_path - source_dir = dir_file_mapping[path] - new_path = destination / source_dir.name / primary_path.relative_to(source_dir) - else: - # Deleted source (pre-moved) -> destination / filename - new_path = destination / primary_path.name - - # Check for conflicts (destination exists) - if new_path.exists(): - if not primary_path.exists(): - # Pre-moved - OK to proceed - plan[primary_path] = new_path + # Is this from a file source or a directory source? + if path_str in file_source_set: + # Direct file source -> destination / filename + new_path = destination / primary_path.name + elif path in dir_file_mapping: + # From directory source -> destination / source_dir.name / relative_path + source_dir = dir_file_mapping[path] + new_path = destination / source_dir.name / primary_path.relative_to(source_dir) else: - raise FileExistsError(f'Cannot move: {new_path} already exists') - else: - plan[primary_path] = new_path + # Deleted source (pre-moved) -> destination / filename + new_path = destination / primary_path.name - # Handle files not in DB - create FileGroups - found_paths = set(fg_by_path.keys()) - missing_db_files = [p for p in all_file_paths if str(p) not in found_paths and p.exists()] - - if missing_db_files: - logger.info(f'build_move_plan_bulk: creating FileGroups for {len(missing_db_files)} files not in DB') - for paths in group_files_by_stem(missing_db_files): - try: - get_primary_file(paths) - fg = FileGroup.from_paths(session, *paths) - session.flush() - - # Determine destination path - if any(str(p) in file_source_set for p in paths): - new_path = destination / fg.primary_path.name + # Check for conflicts (destination exists) + if new_path.exists(): + if not primary_path.exists(): + # Pre-moved - OK to proceed + plan[primary_path] = new_path else: - # Find which directory source this came from - source_dir = None - for p in paths: - if p in dir_file_mapping: - source_dir = dir_file_mapping[p] - break - if source_dir: - new_path = destination / source_dir.name / fg.primary_path.relative_to(source_dir) - else: - new_path = destination / fg.primary_path.name - - if new_path.exists() and fg.primary_path.exists(): raise FileExistsError(f'Cannot move: {new_path} already exists') - plan[fg.primary_path] = new_path + else: + plan[primary_path] = new_path - except NoPrimaryFile: - for file in paths: - fg = FileGroup.from_paths(session, file) + # Handle files not in DB - create FileGroups + found_paths = set(fg_by_path.keys()) + missing_db_files = [p for p in all_file_paths if str(p) not in found_paths and p.exists()] + + if missing_db_files: + logger.info(f'build_move_plan_bulk: creating FileGroups for {len(missing_db_files)} files not in DB') + for paths in group_files_by_stem(missing_db_files): + try: + get_primary_file(paths) + fg = FileGroup.from_paths(session, *paths) session.flush() + + # Determine destination path if any(str(p) in file_source_set for p in paths): new_path = destination / fg.primary_path.name - elif file in dir_file_mapping: - source_dir = dir_file_mapping[file] - new_path = destination / source_dir.name / fg.primary_path.relative_to(source_dir) else: - new_path = destination / fg.primary_path.name + # Find which directory source this came from + source_dir = None + for p in paths: + if p in dir_file_mapping: + source_dir = dir_file_mapping[p] + break + if source_dir: + new_path = destination / source_dir.name / fg.primary_path.relative_to(source_dir) + else: + new_path = destination / fg.primary_path.name + if new_path.exists() and fg.primary_path.exists(): raise FileExistsError(f'Cannot move: {new_path} already exists') plan[fg.primary_path] = new_path - # Add directories to plan for cleanup - for source_dir in dir_sources: - for directory in (i for i in walk(source_dir) if i.is_dir()): - new_directory = destination / source_dir.name / directory.relative_to(source_dir) - plan[directory] = new_directory + except NoPrimaryFile: + for file in paths: + fg = FileGroup.from_paths(session, file) + session.flush() + if any(str(p) in file_source_set for p in paths): + new_path = destination / fg.primary_path.name + elif file in dir_file_mapping: + source_dir = dir_file_mapping[file] + new_path = destination / source_dir.name / fg.primary_path.relative_to(source_dir) + else: + new_path = destination / fg.primary_path.name + if new_path.exists() and fg.primary_path.exists(): + raise FileExistsError(f'Cannot move: {new_path} already exists') + plan[fg.primary_path] = new_path - # Commit to drop the temporary table - session.commit() + # Add directories to plan for cleanup + for source_dir in dir_sources: + for directory in (i for i in walk(source_dir) if i.is_dir()): + new_directory = destination / source_dir.name / directory.relative_to(source_dir) + plan[directory] = new_directory + + except BaseException: + # SQLite has no ON COMMIT DROP; attempt to drop the temp table, but do NOT + # commit — partial plan work must not be persisted on failure. (Outside of + # tests, get_db_session rolls back, which also undoes the CREATE TEMP TABLE; + # the IF NOT EXISTS + DELETE guard above handles any leftover.) + try: + session.execute(text("DROP TABLE IF EXISTS move_sources")) + except Exception: + logger.exception('Failed to drop move_sources temp table') + raise + else: + # Drop the temp table, then commit the plan work (SQLite has no ON COMMIT DROP). + session.execute(text("DROP TABLE IF EXISTS move_sources")) + session.commit() # Sort plan by depth (deepest first) for safe move ordering plan = OrderedDict( @@ -1248,7 +1251,7 @@ class FileWorker: from wrolpi.db import get_db_curs with get_db_curs(commit=True) as curs: for path in deleted_paths: - curs.execute('DELETE FROM directory WHERE path = %s', (str(path.absolute()),)) + curs.execute('DELETE FROM directory WHERE path = ?', (str(path.absolute()),)) # Send appropriate completion event based on refresh scope if is_global_refresh: @@ -1322,8 +1325,8 @@ class FileWorker: curs.execute( '''SELECT primary_path, id FROM file_group - WHERE primary_path = ANY (%s)''', - (all_new_paths,) + WHERE primary_path IN (SELECT value FROM json_each(:paths))''', + {'paths': json.dumps(all_new_paths)} ) existing_path_to_id = {row[0]: row[1] for row in curs.fetchall()} @@ -1338,15 +1341,17 @@ class FileWorker: logger.warning( f'Skipping primary_path update for fg_id={fg_id}: {new_path} already exists') - # Bulk UPDATE using VALUES clause for all valid updates + # Bulk UPDATE using UPDATE...FROM over a JSON array of [id, new_path] pairs if valid_updates: curs.execute( - '''UPDATE file_group AS fg + '''UPDATE file_group SET primary_path = v.new_path - FROM (SELECT unnest(%s::integer[]) AS id, unnest(%s::text[]) AS new_path) AS v - WHERE fg.id = v.id - AND fg.primary_path != v.new_path''', - ([u[0] for u in valid_updates], [u[1] for u in valid_updates]) + FROM (SELECT json_extract(value, '$[0]') AS id, + json_extract(value, '$[1]') AS new_path + FROM json_each(:updates)) AS v + WHERE file_group.id = v.id + AND file_group.primary_path != v.new_path''', + {'updates': json.dumps([[fg_id, new_path] for fg_id, new_path in valid_updates])} ) # Collect all file paths from the diffs (fast operation, no progress tracking needed) @@ -1472,18 +1477,18 @@ class FileWorker: SELECT 'video' AS model_type, v.id AS model_id, fg.id AS fg_id, fg.primary_path FROM video v JOIN file_group fg ON v.file_group_id = fg.id - WHERE fg.id = ANY (%s) + WHERE fg.id IN (SELECT value FROM json_each(:fg_ids)) UNION ALL SELECT 'archive', a.id, fg.id, fg.primary_path FROM archive a JOIN file_group fg ON a.file_group_id = fg.id - WHERE fg.id = ANY (%s) + WHERE fg.id IN (SELECT value FROM json_each(:fg_ids)) UNION ALL SELECT 'doc', d.id, fg.id, fg.primary_path FROM doc d JOIN file_group fg ON d.file_group_id = fg.id - WHERE fg.id = ANY (%s) - ''', (file_group_ids, file_group_ids, file_group_ids)) + WHERE fg.id IN (SELECT value FROM json_each(:fg_ids)) + ''', {'fg_ids': json.dumps(file_group_ids)}) # Group results by model type video_ids_to_delete = [] @@ -1503,30 +1508,39 @@ class FileWorker: # Batch delete each model type if video_ids_to_delete: logger.info(f'Deleting {len(video_ids_to_delete)} Videos whose primary file was removed') + params = {'ids': json.dumps(video_ids_to_delete)} curs.execute(''' UPDATE file_group SET model = NULL - WHERE id IN (SELECT file_group_id FROM video WHERE id = ANY (%s)) - ''', (video_ids_to_delete,)) - curs.execute('DELETE FROM video WHERE id = ANY(%s)', (video_ids_to_delete,)) + WHERE id IN (SELECT file_group_id + FROM video + WHERE id IN (SELECT value FROM json_each(:ids))) + ''', params) + curs.execute('DELETE FROM video WHERE id IN (SELECT value FROM json_each(:ids))', params) if archive_ids_to_delete: logger.info(f'Deleting {len(archive_ids_to_delete)} Archives whose primary file was removed') + params = {'ids': json.dumps(archive_ids_to_delete)} curs.execute(''' UPDATE file_group SET model = NULL - WHERE id IN (SELECT file_group_id FROM archive WHERE id = ANY (%s)) - ''', (archive_ids_to_delete,)) - curs.execute('DELETE FROM archive WHERE id = ANY(%s)', (archive_ids_to_delete,)) + WHERE id IN (SELECT file_group_id + FROM archive + WHERE id IN (SELECT value FROM json_each(:ids))) + ''', params) + curs.execute('DELETE FROM archive WHERE id IN (SELECT value FROM json_each(:ids))', params) if doc_ids_to_delete: logger.info(f'Deleting {len(doc_ids_to_delete)} Docs whose primary file was removed') + params = {'ids': json.dumps(doc_ids_to_delete)} curs.execute(''' UPDATE file_group SET model = NULL - WHERE id IN (SELECT file_group_id FROM doc WHERE id = ANY (%s)) - ''', (doc_ids_to_delete,)) - curs.execute('DELETE FROM doc WHERE id = ANY(%s)', (doc_ids_to_delete,)) + WHERE id IN (SELECT file_group_id + FROM doc + WHERE id IN (SELECT value FROM json_each(:ids))) + ''', params) + curs.execute('DELETE FROM doc WHERE id IN (SELECT value FROM json_each(:ids))', params) def _validate_move_paths( self, @@ -1697,7 +1711,8 @@ class FileWorker: # Batch cleanup DB records for directories that were already deleted if deleted_dirs_for_db: with get_db_curs(commit=True) as curs: - curs.execute('DELETE FROM directory WHERE path = ANY(%s)', (deleted_dirs_for_db,)) + curs.execute('DELETE FROM directory WHERE path IN (SELECT value FROM json_each(:paths))', + {'paths': json.dumps(deleted_dirs_for_db)}) async def _refresh_files_directly( self, diff --git a/wrolpi/flags.py b/wrolpi/flags.py index 26aca9c1..dbc59a3b 100644 --- a/wrolpi/flags.py +++ b/wrolpi/flags.py @@ -1,7 +1,6 @@ import asyncio import contextlib import multiprocessing -import subprocess from datetime import datetime from pathlib import Path @@ -87,15 +86,16 @@ class Flag: def _save(self, value: bool): if self.store_db: - from wrolpi.db import get_db_curs + from wrolpi.db import get_db_session # Store the value of this Flag in it's matching column in the DB. Not all flags will do this. try: - with get_db_curs(commit=True) as curs: + with get_db_session(commit=True) as session: # Upsert the only row in the table. - curs.execute(f''' - INSERT INTO wrolpi_flag (id, {self.name}) VALUES (1, %(value)s) - ON CONFLICT (id) DO UPDATE SET {self.name} = %(value)s - ''', dict(value=value)) + flag_row = session.query(WROLPiFlag).filter_by(id=1).one_or_none() + if not flag_row: + flag_row = WROLPiFlag(id=1) + session.add(flag_row) + setattr(flag_row, self.name, value) except Exception as e: logger.critical(f'Unable to save flag! {repr(self)}', exc_info=e) @@ -108,6 +108,9 @@ class Flag: # Set by `check_db_is_up`. db_up = Flag('db_up') +# The database file was written by a NEWER WROLPi than this one (drive moved from another system). +# While set, the database must not be written; upgrade this WROLPi instead. +db_version_mismatch = Flag('db_version_mismatch') # Outdated Zims need to be removed. outdated_zims = Flag('outdated_zims', store_db=True) # Used to disable or enable downloading when Internet is down. @@ -236,26 +239,28 @@ def _destinations_have_mounted_storage() -> bool: def check_db_is_up(): - """Attempts to connect to the database, sets flags.db_up if successful.""" - from wrolpi.db import get_db_curs, get_db_args + """Checks that the database file exists and is initialized, sets flags.db_up if so. - db_args = get_db_args() - db_host = db_args['host'] - if db_host != '127.0.0.1': - # Check if database host is up before attempting connection. This will fail faster than SQLAlchemy timeout. - try: - cmd = ['ping', '-w', '1', '-c', '1', db_host] - subprocess.check_call(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - except Exception as e: - logger.debug(f'Unable to resolve database host {db_host}', exc_info=e) - db_up.clear() - return + The database lives inside the media directory, so "DB down" usually means the media drive + is not mounted (or the DB has not been created/migrated yet).""" + import sqlite3 + from wrolpi.db import get_db_file + from wrolpi.db_bootstrap import media_directory_is_unmounted_production try: - with get_db_curs() as curs: - curs.execute('SELECT * FROM alembic_version') - # If we get here, the database is up! - db_up.set() + if media_directory_is_unmounted_production(): + # Any DB file at the media path is a root-filesystem shadow, not the real database. + db_up.clear() + return + db_file = get_db_file() + if not db_file.is_file() or db_version_mismatch.is_set(): + db_up.clear() + return + # Read-only URI probe; a plain connect would create an empty database file. + with contextlib.closing(sqlite3.connect(f'file:{db_file}?mode=ro', uri=True)) as conn: + conn.execute('SELECT version_num FROM alembic_version').fetchone() + # If we get here, the database is up! + db_up.set() except Exception as e: logger.debug(f'DB is not up', exc_info=e) db_up.clear() diff --git a/wrolpi/fts.py b/wrolpi/fts.py new file mode 100644 index 00000000..aeca804f --- /dev/null +++ b/wrolpi/fts.py @@ -0,0 +1,304 @@ +"""SQLite FTS5 full-text search for WROLPi. + +This module owns everything FTS5: the DDL for the external-content tables and their sync +triggers, the websearch → FTS5 query translator, SQL fragment builders used by the search +call sites, and headline/snippet helpers. + +No other module may contain FTS5 syntax (MATCH, bm25, snippet, highlight). + +Design notes: + * `file_group_fts` mirrors `file_group.a_text..d_text` (external content, so the text is + stored once). `content_rowid` requires `file_group.id` to be a true rowid alias + ("INTEGER PRIMARY KEY" exactly) — see the model. + * Ranking uses bm25 weights 10/4/2/1 which reproduce Postgres' A/B/C/D tsvector weighting. + FTS5 `rank` is smaller-is-better (negative); this module always exposes `-rank AS ts_rank` + so consumers keep the "higher is better" contract of `ts_rank`. + * The "fast" search path (previously the `textsearch_abc` column) is a column filter that + excludes `d_text`; the "deep" path searches all columns. +""" +import contextlib +import dataclasses +import re +import sqlite3 +from typing import List, Optional, Tuple + +FILE_GROUP_FTS_COLUMNS = ('a_text', 'b_text', 'c_text', 'd_text') +# The fast (non-deep) path searches these columns only. +ABC_COLUMNS = ('a_text', 'b_text', 'c_text') + +# Reproduces ts_rank's default weights {D=0.1, C=0.2, B=0.4, A=1.0} (scaled x10), in FTS5 +# column order (a_text..d_text). Must have exactly one weight per FTS column. +FILE_GROUP_BM25_WEIGHTS = 'bm25(10.0, 4.0, 2.0, 1.0)' + +TOKENIZER = 'porter unicode61' + +# External-content FTS5 tables + the triggers that keep them in sync. +# +# The 'delete' command rows in the triggers must reproduce the OLD values exactly; never use +# INSERT OR REPLACE on file_group/doc_section (its implicit delete corrupts the FTS index), +# always ON CONFLICT ... DO UPDATE. +FTS_DDL = [ + f''' + CREATE VIRTUAL TABLE IF NOT EXISTS file_group_fts USING fts5( + a_text, b_text, c_text, d_text, + content='file_group', + content_rowid='id', + tokenize='{TOKENIZER}' + ) + ''', + # Persist the weighted rank so bare `ORDER BY rank` uses the A/B/C/D weighting. + f''' + INSERT INTO file_group_fts(file_group_fts, rank) VALUES('rank', '{FILE_GROUP_BM25_WEIGHTS}') + ''', + ''' + CREATE TRIGGER IF NOT EXISTS file_group_fts_ai AFTER INSERT ON file_group BEGIN + INSERT INTO file_group_fts(rowid, a_text, b_text, c_text, d_text) + VALUES (new.id, new.a_text, new.b_text, new.c_text, new.d_text); + END + ''', + ''' + CREATE TRIGGER IF NOT EXISTS file_group_fts_ad AFTER DELETE ON file_group BEGIN + INSERT INTO file_group_fts(file_group_fts, rowid, a_text, b_text, c_text, d_text) + VALUES ('delete', old.id, old.a_text, old.b_text, old.c_text, old.d_text); + END + ''', + ''' + CREATE TRIGGER IF NOT EXISTS file_group_fts_au + AFTER UPDATE OF a_text, b_text, c_text, d_text ON file_group BEGIN + INSERT INTO file_group_fts(file_group_fts, rowid, a_text, b_text, c_text, d_text) + VALUES ('delete', old.id, old.a_text, old.b_text, old.c_text, old.d_text); + INSERT INTO file_group_fts(rowid, a_text, b_text, c_text, d_text) + VALUES (new.id, new.a_text, new.b_text, new.c_text, new.d_text); + END + ''', + f''' + CREATE VIRTUAL TABLE IF NOT EXISTS doc_section_fts USING fts5( + content, + content='doc_section', + content_rowid='id', + tokenize='{TOKENIZER}' + ) + ''', + ''' + CREATE TRIGGER IF NOT EXISTS doc_section_fts_ai AFTER INSERT ON doc_section BEGIN + INSERT INTO doc_section_fts(rowid, content) VALUES (new.id, new.content); + END + ''', + ''' + CREATE TRIGGER IF NOT EXISTS doc_section_fts_ad AFTER DELETE ON doc_section BEGIN + INSERT INTO doc_section_fts(doc_section_fts, rowid, content) VALUES ('delete', old.id, old.content); + END + ''', + ''' + CREATE TRIGGER IF NOT EXISTS doc_section_fts_au AFTER UPDATE OF content ON doc_section BEGIN + INSERT INTO doc_section_fts(doc_section_fts, rowid, content) VALUES ('delete', old.id, old.content); + INSERT INTO doc_section_fts(rowid, content) VALUES (new.id, new.content); + END + ''', +] + +# FTS5 shadow tables (and the virtual tables themselves); excluded from alembic autogenerate. +FTS_TABLE_PREFIXES = ('file_group_fts', 'doc_section_fts') + + +_ITEM_RE = re.compile(r'-?"[^"]*"?|\S+') +_WORD_RE = re.compile(r'\w+', re.UNICODE) + + +def _render_phrase(words: List[str]) -> str: + """Render sanitized words as a (safe) FTS5 phrase: every term is double-quoted.""" + return '"' + ' '.join(words) + '"' + + +def translate_websearch(search_str: Optional[str], columns: Optional[Tuple[str, ...]] = None) -> Optional[str]: + """Translate a websearch-style query (Postgres `websearch_to_tsquery` semantics) to FTS5. + + Supported semantics: bare words are ANDed, "double quotes" form phrases, `-term` excludes, + a bare OR separates groups. All user tokens end up inside double-quotes so no FTS5 syntax + can be injected; the output either parses or is None (nothing usable to search). + + Punctuation is stripped from terms (`C++` -> `"C"`, `e.g.` -> the phrase `"e g"`). This is + NOT lossy relative to the index: the unicode61 tokenizer strips the same punctuation when + indexing, so `C++` is stored as the token `c` — exactly as Postgres's `websearch_to_tsquery` + ('C++' -> 'c') and `to_tsvector` did. Punctuation is unsearchable under either engine's + default tokenizer; matching it literally would require a custom FTS5 tokenizer. + + `columns` restricts matching to the given FTS columns (the fast/abc path). + + >>> translate_websearch('two words') + '("two" AND "words")' + >>> translate_websearch('"a phrase" -excluded') + '(("a phrase") NOT "excluded")' + >>> translate_websearch('cat OR dog') + '("cat") OR ("dog")' + """ + if not search_str or not search_str.strip(): + return None + + # Lex into items: quoted phrases (optionally negated) or bare tokens. + items = [] # (negated: bool, words: List[str]) + for token in _ITEM_RE.findall(search_str): + negated = token.startswith('-') + if negated: + token = token[1:] + words = _WORD_RE.findall(token) + if not words: + continue # pure punctuation + items.append((negated, words)) + + if not items: + return None + + # Split on OR (websearch: a bare, un-negated, unquoted "or"; we accept any lone "or" token). + groups: List[List[Tuple[bool, List[str]]]] = [[]] + for negated, words in items: + if not negated and len(words) == 1 and words[0].lower() == 'or': + if groups[-1]: + groups.append([]) + continue + groups[-1].append((negated, words)) + + rendered_groups = [] + for group in groups: + positives = [_render_phrase(words) for negated, words in group if not negated] + negatives = [_render_phrase(words) for negated, words in group if negated] + if not positives: + # FTS5 NOT is binary; a group with only negatives is not expressible. Postgres + # would match "everything not containing the term" -- accepted divergence: skip. + continue + expr = '(' + ' AND '.join(positives) + ')' + for negative in negatives: + expr = f'({expr} NOT {negative})' + rendered_groups.append(expr) + + if not rendered_groups: + return None + + result = ' OR '.join(f'({group})' for group in rendered_groups) + if columns: + result = '{' + ' '.join(columns) + '} : (' + result + ')' + return result + + +@dataclasses.dataclass +class FileGroupSearch: + """SQL fragments for joining file_group against its FTS5 table. + + Use in a query like: + SELECT fg.id, {rank_select} FROM file_group fg {join} WHERE {where} ORDER BY 2 DESC + """ + join: str # JOIN clause linking file_group_fts to file_group (aliased fg) + where: str # MATCH condition + rank_select: str # positive higher-is-better rank, SELECTable as ts_rank + params: dict # bind parameters used by the fragments + + +def file_group_search(search_str: str, deep: bool = False, + fg_alias: str = 'fg', param_name: str = 'fts_match') -> Optional[FileGroupSearch]: + """Build the FTS5 join/where/rank fragments for a file_group search. + + Returns None when there is nothing usable to search; callers must then treat the request + like an empty search (no rank column, no MATCH condition).""" + columns = None if deep else ABC_COLUMNS + match = translate_websearch(search_str, columns=columns) + if match is None: + return None + return FileGroupSearch( + join=f'JOIN file_group_fts fts ON fts.rowid = {fg_alias}.id', + where=f'fts.file_group_fts MATCH :{param_name}', + rank_select='-fts.rank AS ts_rank', + params={param_name: match}, + ) + + +def file_group_search_join(search_str: str, deep: bool = False, headlines: bool = False, + fg_alias: str = 'fg', param_name: str = 'fts_match', + start: str = '', stop: str = '', tokens: int = 16) -> Optional[FileGroupSearch]: + """Subquery-join form of `file_group_search`. + + SQLite refuses FTS5 auxiliary functions (snippet) in a query that also uses window functions + (e.g. COUNT(*) OVER()), so the rank and snippets are computed inside a subquery and joined. + The returned `join` both filters (INNER JOIN on the MATCH) and provides `fts.ts_rank` (and + `fts.b_headline`/`c_headline`/`d_headline` when `headlines`); `where` is empty.""" + columns = None if deep else ABC_COLUMNS + match = translate_websearch(search_str, columns=columns) + if match is None: + return None + snippet_selects = '' + if headlines: + snippet_selects = (f", snippet(file_group_fts, 1, '{start}', '{stop}', '…', {tokens}) AS b_headline" + f", snippet(file_group_fts, 2, '{start}', '{stop}', '…', {tokens}) AS c_headline" + f", snippet(file_group_fts, 3, '{start}', '{stop}', '…', {tokens}) AS d_headline") + join = (f'JOIN (SELECT rowid, -rank AS ts_rank{snippet_selects} FROM file_group_fts ' + f'WHERE file_group_fts MATCH :{param_name}) fts ON fts.rowid = {fg_alias}.id') + return FileGroupSearch(join=join, where='', rank_select='fts.ts_rank AS ts_rank', + params={param_name: match}) + + +def file_group_headline_selects(start: str = '', stop: str = '', tokens: int = 16) -> str: + """SELECT fragments producing the b/c/d headline columns from the joined FTS table. + + `title_headline` is not produced here: `title` is not an FTS column; use `headline_texts` + on the result rows (matches the old ts_headline-over-title behavior).""" + return f''', + snippet(file_group_fts, 1, '{start}', '{stop}', '…', {tokens}) AS b_headline, + snippet(file_group_fts, 2, '{start}', '{stop}', '…', {tokens}) AS c_headline, + snippet(file_group_fts, 3, '{start}', '{stop}', '…', {tokens}) AS d_headline''' + + +def headline_texts(entries: List[Optional[str]], search_str: str, + start: str = '', stop: str = '', tokens: int = 8) -> List[Tuple[str, float]]: + """Highlight `search_str` matches in ad-hoc strings; returns [(headline, rank), ...]. + + Replaces Postgres `ts_headline`/`ts_rank` over arbitrary content (Zim search results, + FileGroup titles). Uses a throwaway in-memory FTS5 table so stemming matches the real + search index. Non-matching entries yield the start of the text with rank 0.""" + + def _lead(text: Optional[str]) -> str: + words = (text or '').split() + return ' '.join(words[:tokens]) + + results = [(_lead(text), 0.0) for text in entries] + + match = translate_websearch(search_str) + if match is None or not entries: + return results + + with contextlib.closing(sqlite3.connect(':memory:')) as conn: + conn.execute(f"CREATE VIRTUAL TABLE h USING fts5(content, tokenize='{TOKENIZER}')") + conn.executemany('INSERT INTO h(rowid, content) VALUES (?, ?)', + [(idx, text or '') for idx, text in enumerate(entries)]) + curs = conn.execute( + f"SELECT rowid, snippet(h, 0, ?, ?, '…', ?), rank FROM h WHERE h MATCH ?", + (start, stop, tokens, match)) + for rowid, headline, rank in curs.fetchall(): + results[rowid] = (headline, -rank) + + return results + + +def rebuild_fts(curs): + """Rebuild the FTS5 indexes from their content tables (e.g. after a bulk import).""" + curs.execute("INSERT INTO file_group_fts(file_group_fts) VALUES('rebuild')") + curs.execute("INSERT INTO doc_section_fts(doc_section_fts) VALUES('rebuild')") + + +def optimize_fts(curs): + """Merge FTS5 b-trees for faster queries; call after large refreshes.""" + curs.execute("INSERT INTO file_group_fts(file_group_fts) VALUES('optimize')") + curs.execute("INSERT INTO doc_section_fts(doc_section_fts) VALUES('optimize')") + + +def fts_integrity_ok(curs) -> bool: + """Verify the FTS5 indexes match their content tables.""" + try: + curs.execute("INSERT INTO file_group_fts(file_group_fts, rank) VALUES('integrity-check', 1)") + curs.execute("INSERT INTO doc_section_fts(doc_section_fts, rank) VALUES('integrity-check', 1)") + return True + except sqlite3.DatabaseError: + return False + except Exception as e: + # SQLAlchemy-wrapped DatabaseError. + if 'malformed' in str(e) or 'corrupt' in str(e).lower(): + return False + raise diff --git a/wrolpi/schema_ddl.py b/wrolpi/schema_ddl.py new file mode 100644 index 00000000..b309b012 --- /dev/null +++ b/wrolpi/schema_ddl.py @@ -0,0 +1,138 @@ +"""Raw SQLite DDL shared by the Alembic baseline and the test database. + +SQLAlchemy models (`Base.metadata`) define the tables; this module holds everything the models +cannot express: triggers and the FTS5 full-text-search tables (see `wrolpi.fts`). + +`install_raw_ddl` is executed by BOTH: + * the Alembic baseline migration (production databases), and + * `wrolpi.conftest.test_db` (test databases, after `Base.metadata.create_all`), +so tests exercise exactly the schema production runs. + +All statements are idempotent (IF NOT EXISTS). +""" + +# Triggers maintaining the summary columns `channel.video_count`, `channel.total_size`, +# `channel.minimum_frequency` and `file_group.effective_datetime`. +# +# These are re-authored from the old PL/pgSQL triggers (see git history of alembic/versions/). +# The old triggers had known bugs which are fixed here: +# * video_count used NEW.channel_id on DELETE (always NULL in Postgres), so counts never +# decremented, and did not fire on UPDATE OF channel_id (re-parenting). +# * total_size used NEW.id (a video/file_group id) as the channel id. +TRIGGER_DDL = [ + # channel.video_count + channel.total_size follow the videos in the channel. + ''' + CREATE TRIGGER IF NOT EXISTS video_insert_channel_summary + AFTER INSERT ON video WHEN new.channel_id IS NOT NULL + BEGIN + UPDATE channel SET + video_count = (SELECT COUNT(*) FROM video WHERE channel_id = channel.id), + total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM video v + LEFT JOIN file_group fg ON v.file_group_id = fg.id + WHERE v.channel_id = channel.id) + WHERE id = new.channel_id; + END + ''', + ''' + CREATE TRIGGER IF NOT EXISTS video_delete_channel_summary + AFTER DELETE ON video WHEN old.channel_id IS NOT NULL + BEGIN + UPDATE channel SET + video_count = (SELECT COUNT(*) FROM video WHERE channel_id = channel.id), + total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM video v + LEFT JOIN file_group fg ON v.file_group_id = fg.id + WHERE v.channel_id = channel.id) + WHERE id = old.channel_id; + END + ''', + # Re-parenting a video (or changing its FileGroup) updates both affected channels. + ''' + CREATE TRIGGER IF NOT EXISTS video_update_channel_summary + AFTER UPDATE OF channel_id, file_group_id ON video + BEGIN + UPDATE channel SET + video_count = (SELECT COUNT(*) FROM video WHERE channel_id = channel.id), + total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM video v + LEFT JOIN file_group fg ON v.file_group_id = fg.id + WHERE v.channel_id = channel.id) + WHERE id IN (old.channel_id, new.channel_id); + END + ''', + # A FileGroup's size changed; update the Channel of the Video which owns it (if any). + ''' + CREATE TRIGGER IF NOT EXISTS file_group_size_channel_summary + AFTER UPDATE OF size ON file_group WHEN old.size IS NOT new.size + BEGIN + UPDATE channel SET + total_size = (SELECT COALESCE(SUM(fg.size), 0) FROM video v + LEFT JOIN file_group fg ON v.file_group_id = fg.id + WHERE v.channel_id = channel.id) + WHERE id = (SELECT channel_id FROM video WHERE file_group_id = new.id); + END + ''', + # channel.minimum_frequency follows the Downloads of the Channel's Collection. + ''' + CREATE TRIGGER IF NOT EXISTS download_insert_minimum_frequency + AFTER INSERT ON download WHEN new.collection_id IS NOT NULL + BEGIN + UPDATE channel SET + minimum_frequency = (SELECT MIN(frequency) FROM download + WHERE collection_id = channel.collection_id) + WHERE collection_id = new.collection_id; + END + ''', + ''' + CREATE TRIGGER IF NOT EXISTS download_update_minimum_frequency + AFTER UPDATE OF frequency, collection_id ON download + BEGIN + UPDATE channel SET + minimum_frequency = (SELECT MIN(frequency) FROM download + WHERE collection_id = channel.collection_id) + WHERE collection_id IN (old.collection_id, new.collection_id); + END + ''', + ''' + CREATE TRIGGER IF NOT EXISTS download_delete_minimum_frequency + AFTER DELETE ON download WHEN old.collection_id IS NOT NULL + BEGIN + UPDATE channel SET + minimum_frequency = (SELECT MIN(frequency) FROM download + WHERE collection_id = channel.collection_id) + WHERE collection_id = old.collection_id; + END + ''', + # file_group.effective_datetime = COALESCE(published_datetime, download_datetime). + # Also maintained by an ORM event (wrolpi.files.models.update_effective_datetime); these + # triggers cover raw-SQL writers. The WHEN clause avoids rewriting rows the ORM already set. + ''' + CREATE TRIGGER IF NOT EXISTS file_group_effective_datetime_insert + AFTER INSERT ON file_group + WHEN new.effective_datetime IS NOT COALESCE(new.published_datetime, new.download_datetime) + BEGIN + UPDATE file_group + SET effective_datetime = COALESCE(new.published_datetime, new.download_datetime) + WHERE id = new.id; + END + ''', + ''' + CREATE TRIGGER IF NOT EXISTS file_group_effective_datetime_update + AFTER UPDATE OF published_datetime, download_datetime ON file_group + WHEN new.effective_datetime IS NOT COALESCE(new.published_datetime, new.download_datetime) + BEGIN + UPDATE file_group + SET effective_datetime = COALESCE(new.published_datetime, new.download_datetime) + WHERE id = new.id; + END + ''', +] + + +def install_raw_ddl(conn): + """Install all raw DDL (triggers + FTS5) on a SQLite database. + + `conn` may be a SQLAlchemy Connection (e.g. `op.get_bind()` in Alembic) or a raw + `sqlite3.Connection`. Idempotent.""" + from wrolpi import fts + + for statement in [*TRIGGER_DDL, *fts.FTS_DDL]: + conn.execute(statement) diff --git a/wrolpi/tags.py b/wrolpi/tags.py index c74e660d..cac79175 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -14,7 +14,7 @@ from wrolpi import dates, flags from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_directory, background_task, \ get_relative_to_media_directory, is_valid_hex_color, walk, INVALID_FILE_CHARS, get_wrolpi_config from wrolpi.dates import TZDateTime -from wrolpi.db import get_db_curs, get_db_session +from wrolpi.db import get_db_curs, get_db_session, named_placeholders from wrolpi.downloader import save_downloads_config from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, FileWorkerConflict, NoPrimaryFile from wrolpi.events import Events @@ -449,6 +449,7 @@ class TagsConfig(ConfigFile): session.flush([file_group, ]) except NoPrimaryFile: logger.error(f'Failed to tag {absolute_path}') + self.import_skipped += 1 continue if tag and file_group: @@ -464,8 +465,10 @@ class TagsConfig(ConfigFile): need_commit = True elif not file_group: logger.warning(f'Cannot find FileGroup for {repr(str(primary_path))}') + self.import_skipped += 1 elif not tag: logger.warning(f'Cannot find Tag for {repr(str(primary_path))}') + self.import_skipped += 1 # Tag all Zim entries. if config.tag_zims: @@ -489,6 +492,7 @@ class TagsConfig(ConfigFile): possible_zims = {i: j for i, j in zims_by_path.items() if i.name.startswith(name)} if not possible_zims: logger.warning(f'Cannot find Zim for {repr(str(zim_path))}') + self.import_skipped += 1 continue zim = zims_by_path[sorted(possible_zims.keys())[-1]] new_zim_path = get_relative_to_media_directory(media_directory / zim.path) @@ -508,6 +512,7 @@ class TagsConfig(ConfigFile): need_commit = True elif not tag: logger.warning(f'Cannot find Tag for {repr(str(zim_path))}') + self.import_skipped += 1 # Delete missing Tags last in case they are used above. if config.tags: @@ -838,7 +843,7 @@ def get_recent_tags(limit: int = 5) -> List[str]: FROM tag_zim) AS combined GROUP BY tag_id ORDER BY latest DESC - LIMIT %(limit)s) AS recent + LIMIT :limit) AS recent JOIN tag t ON t.id = recent.tag_id ORDER BY recent.latest DESC ''', dict(limit=limit)) @@ -849,22 +854,25 @@ def get_overlapping_tags(tag_names: Union[str, List[str]], limit: int = 5) -> Li if isinstance(tag_names, str): tag_names = [tag_names] - limit_clause = 'LIMIT %(limit)s' if limit is not None else '' + limit_clause = 'LIMIT :limit' if limit is not None else '' + + params = dict(tag_count=len(tag_names), limit=limit) + tag_names_placeholders = named_placeholders('tag_name', tag_names, params) with get_db_curs() as curs: curs.execute(f''' - WITH target_tags AS (SELECT id FROM tag WHERE name = ANY(%(tag_names)s::TEXT[])), + WITH target_tags AS (SELECT id FROM tag WHERE name IN ({tag_names_placeholders})), file_matching AS ( SELECT tf.file_group_id FROM tag_file tf WHERE tf.tag_id IN (SELECT id FROM target_tags) GROUP BY tf.file_group_id - HAVING COUNT(DISTINCT tf.tag_id) = %(tag_count)s + HAVING COUNT(DISTINCT tf.tag_id) = :tag_count ), zim_matching AS ( SELECT tz.zim_id, tz.zim_entry FROM tag_zim tz WHERE tz.tag_id IN (SELECT id FROM target_tags) GROUP BY tz.zim_id, tz.zim_entry - HAVING COUNT(DISTINCT tz.tag_id) = %(tag_count)s + HAVING COUNT(DISTINCT tz.tag_id) = :tag_count ), file_overlap AS ( SELECT tf.tag_id, COUNT(*) AS cnt FROM tag_file tf @@ -882,7 +890,7 @@ def get_overlapping_tags(tag_names: Union[str, List[str]], limit: int = 5) -> Li ) AS all_overlap GROUP BY tag_id ) SELECT t.name FROM combined c JOIN tag t ON t.id = c.tag_id ORDER BY c.total DESC {limit_clause} - ''', dict(tag_names=tag_names, tag_count=len(tag_names), limit=limit)) + ''', params) return [row['name'] for row in curs.fetchall()] @@ -915,16 +923,17 @@ def tag_names_to_file_group_sub_select(tag_names: List[str], params: dict) -> Tu if not tag_names: return '', params - # This select gets an array of FileGroup.id's which are tagged with the provided tag names. - sub_select = ''' + # This select gets the FileGroup.id's which are tagged with the provided tag names. + tag_names_placeholders = named_placeholders('tag_name', tag_names, params) + sub_select = f''' SELECT tf.file_group_id FROM tag_file tf - LEFT JOIN tag t on t.id = tf.tag_id - GROUP BY file_group_id + JOIN tag t ON t.id = tf.tag_id AND t.name IN ({tag_names_placeholders}) + GROUP BY tf.file_group_id -- Match only FileGroups that have at least all the Tag names. - HAVING array_agg(t.name)::TEXT[] @> %(tag_names)s::TEXT[] \ + HAVING COUNT(DISTINCT t.name) = :tag_names_count \ ''' - params['tag_names'] = tag_names + params['tag_names_count'] = len(tag_names) return sub_select, params @@ -942,7 +951,7 @@ def tag_append_sub_select_where(wheres: List[str], params: dict, tag_names: List tags_sub_select = 'SELECT file_group_id FROM tag_file' else: tags_sub_select, params = tag_names_to_file_group_sub_select(tag_names, params) - wheres.append(f'fg.id = ANY({tags_sub_select})') + wheres.append(f'fg.id IN ({tags_sub_select})') return wheres, params @@ -950,27 +959,28 @@ def tag_names_to_zim_sub_select(tag_names: List[str], zim_id: int = None) -> Tup if not tag_names: return '', dict() - params = dict(tag_names=tag_names) + params = dict(tag_names_count=len(tag_names)) + tag_names_placeholders = named_placeholders('tag_name', tag_names, params) if zim_id: - stmt = ''' + stmt = f''' SELECT tz.zim_id, tz.zim_entry FROM tag_zim tz - LEFT JOIN tag t on tz.tag_id = t.id - WHERE tz.zim_id = %(zim_id)s + JOIN tag t ON t.id = tz.tag_id AND t.name IN ({tag_names_placeholders}) + WHERE tz.zim_id = :zim_id GROUP BY tz.zim_id, tz.zim_entry -- Match only TagZimEntries that have all the Tag names. - HAVING array_agg(t.name)::TEXT[] @> %(tag_names)s::TEXT[] \ + HAVING COUNT(DISTINCT t.name) = :tag_names_count \ ''' params['zim_id'] = zim_id else: - stmt = ''' + stmt = f''' SELECT tz.zim_id, tz.zim_entry FROM tag_zim tz - LEFT JOIN tag t on tz.tag_id = t.id + JOIN tag t ON t.id = tz.tag_id AND t.name IN ({tag_names_placeholders}) GROUP BY tz.zim_id, tz.zim_entry -- Match only TagZimEntries that have all the Tag names. - HAVING array_agg(t.name)::TEXT[] @> %(tag_names)s::TEXT[] \ + HAVING COUNT(DISTINCT t.name) = :tag_names_count \ ''' return stmt, params diff --git a/wrolpi/test/common.py b/wrolpi/test/common.py index 3e4f981c..128163ce 100644 --- a/wrolpi/test/common.py +++ b/wrolpi/test/common.py @@ -6,13 +6,11 @@ from functools import partialmethod from itertools import zip_longest from typing import List -import mock import pytest from wrolpi.api_utils import CustomJSONEncoder from wrolpi.common import get_media_directory from wrolpi.conftest import test_db, async_client # noqa -from wrolpi.db import postgres_engine from wrolpi.vars import CIRCLECI, IS_MACOS TEST_CONFIG_PATH = tempfile.NamedTemporaryFile(mode='rt', delete=False) @@ -22,34 +20,6 @@ skip_macos = pytest.mark.skipif(IS_MACOS, reason='This test is not supported on only_macos = pytest.mark.skipif(not IS_MACOS, reason='This test is only supported on MacOS') -def wrap_test_db(func): - """ - Wrap a test so that when calling wrolpi.common.get_db, it returns a testing database cloned from the api - template. - """ - - def wrapped(*a, **kw): - test_engine, session = test_db() - - def fake_get_db_session(): - """Get the testing db""" - return test_engine, session - - try: - with mock.patch('wrolpi.db._get_db_session', fake_get_db_session): - # Run the test. - result = func(*a, **kw) - return result - finally: - session.rollback() - session.close() - test_engine.dispose() - conn = postgres_engine.connect() - conn.execute(f'DROP DATABASE IF EXISTS {test_engine.engine.url.database}') - - return wrapped - - class PytestCase: """ Replicate unittest.TestCase methods to bridge the gap between it and Pytest. diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 6530eb64..959057f7 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -664,8 +664,12 @@ def test_url_strip_host(url, expected): @pytest.mark.parametrize( 'search_str,expected', [ - ('jump', [('jumps over the lazy brown', 0.06079271), ('jumped over the lazy', 0.06079271)]), - ('brown', [('over the lazy brown', 0.06079271), ('The fox jumped over the lazy dog', 0.0)]), + # Matches are highlighted (stemmed: jump matches jumps/jumped); non-matches return the + # start of the entry with a rank of 0. + ('jump', [('The fox jumps over the lazy brown dog.', True), + ('The fox jumped over the lazy dog.', True)]), + ('brown', [('The fox jumps over the lazy brown dog.', True), + ('The fox jumped over the lazy dog.', False)]), ] ) def test_extract_headlines(test_session, search_str, expected): @@ -673,7 +677,9 @@ def test_extract_headlines(test_session, search_str, expected): 'The fox jumps over the lazy brown dog.', 'The fox jumped over the lazy dog.', ] - assert common.extract_headlines(entries, search_str) == expected + results = common.extract_headlines(entries, search_str) + # Compare the headline text, and whether the entry matched (exact ranks are engine noise). + assert [(headline, rank > 0) for headline, rank in results] == expected def test_extract_html_text(): diff --git a/wrolpi/test/test_config_partial_import.py b/wrolpi/test/test_config_partial_import.py new file mode 100644 index 00000000..10fd5380 --- /dev/null +++ b/wrolpi/test/test_config_partial_import.py @@ -0,0 +1,161 @@ +"""A partial (lossy) config import must not allow a dump to overwrite the config file. + +The bug: importers that skip entries they cannot resolve (a file not yet indexed, a missing +Tag/Zim, an invalid domain) still mark the import successful. A later `dump_config` regenerates +the YAML from the (incomplete) database and permanently drops the skipped entries. After the +SQLite cutover the database is rebuilt from configs, so this would become permanent data loss. + +The fix: `ConfigFile.import_skipped` counts entries skipped during import; `ConfigFile.save()` +refuses to overwrite an existing config while `import_skipped` is non-zero (unless +`overwrite=True`). A later complete import resets the counter and dumps resume. +""" +from copy import deepcopy + +import pytest + +from modules.archive.lib import get_domains_config +from wrolpi.collections.config import get_playlists_config +from wrolpi.collections.models import Collection +from wrolpi.tags import get_tags_config, Tag + + +TAGS_YAML = '''version: 1 +tags: + important: + color: '#ff0000' +tag_files: +- - important + - docs/report.pdf + - '2026-01-01T00:00:00' +tag_zims: [] +''' + + +@pytest.mark.asyncio +async def test_tags_partial_import_blocks_dump(async_client, test_session, test_directory, test_tags_config, + await_switches): + """A tags.yaml entry for a file that is not indexed is skipped; dumps must not drop it.""" + config = get_tags_config() + file = config.get_file() + file.write_text(TAGS_YAML) + + # The tagged file does not exist, so the TagFile cannot be restored yet. + config.import_config() + assert config.successful_import is True + assert config.import_skipped == 1 + # The Tag itself was imported. + assert test_session.query(Tag).filter_by(name='important').count() == 1 + + # A background dump (save_tags_config switch) must not overwrite the config. + before = file.read_text() + config.dump_config() + await await_switches() + assert file.read_text() == before, 'A dump after a partial import must not modify the config file' + + # A direct save also refuses. + with pytest.raises(RuntimeError, match='skipped'): + config.save() + + # The escape hatch still works. + config.save(overwrite=True) + + +@pytest.mark.asyncio +async def test_tags_import_recovers_after_file_indexed(async_client, test_session, test_directory, + test_tags_config, await_switches): + """Once the skipped file can be resolved, a re-import resets the counter and dumps resume.""" + config = get_tags_config() + file = config.get_file() + file.write_text(TAGS_YAML) + + config.import_config() + assert config.import_skipped == 1 + + # Create the file; a re-import (e.g. after a refresh) now resolves it. + report = test_directory / 'docs/report.pdf' + report.parent.mkdir(parents=True) + report.write_bytes(b'%PDF-1.4 hello') + config.import_config() + assert config.import_skipped == 0 + assert config.successful_import is True + + # Dumps work again, and the previously-skipped entry is preserved. + config.dump_config() + await await_switches() + data = config.read_config_file(file) + assert ['important', 'docs/report.pdf'] in [i[:2] for i in data['tag_files']] + + +@pytest.fixture +def playlists_config(async_client): + config = get_playlists_config() + config._config = deepcopy(config.default_config) + return config + + +PLAYLISTS_YAML = '''version: 1 +playlists: + - name: Survival + items: + - file: guide.pdf + - url: /map?lat=1&lon=2&z=3 +''' + + +@pytest.mark.asyncio +async def test_playlists_partial_import_blocks_dump(test_session, test_directory, playlists_config): + """A playlist file item that is not indexed is skipped; dumps must not drop it.""" + file = playlists_config.get_file() + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(PLAYLISTS_YAML) + + playlists_config.import_config() + assert playlists_config.successful_import is True + assert playlists_config.import_skipped == 1 + collection = test_session.query(Collection).filter_by(name='Survival', kind='playlist').one() + assert [i.item_kind for i in collection.items] == ['url'] + + before = file.read_text() + with pytest.raises(RuntimeError, match='skipped'): + playlists_config.dump_config() + assert file.read_text() == before, 'A dump after a partial import must not modify the config file' + + # Once the file is indexed, a re-import restores the item and dumps resume. + from wrolpi.files.models import FileGroup + pdf = test_directory / 'guide.pdf' + pdf.write_bytes(b'%PDF-1.4 x') + FileGroup.from_paths(test_session, pdf) + test_session.commit() + + playlists_config.import_config() + assert playlists_config.import_skipped == 0 + playlists_config.dump_config() + data = playlists_config.read_config_file(file) + items = data['playlists'][0]['items'] + assert dict(file='guide.pdf') in [dict(file=i['file']) for i in items if 'file' in i] + + +DOMAINS_YAML = '''version: 1 +collections: + - name: example.com + - name: invaliddomain +''' + + +@pytest.mark.asyncio +async def test_domains_partial_import_blocks_dump(async_client, test_session, test_directory): + """A domain entry with an invalid name is skipped; dumps must not drop it.""" + config = get_domains_config() + file = config.get_file() + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(DOMAINS_YAML) + + config.import_config() + assert config.successful_import is True + assert config.import_skipped == 1 + assert test_session.query(Collection).filter_by(kind='domain').count() == 1 + + before = file.read_text() + with pytest.raises(RuntimeError, match='skipped'): + config.dump_config() + assert file.read_text() == before, 'A dump after a partial import must not modify the config file' diff --git a/wrolpi/test/test_dates.py b/wrolpi/test/test_dates.py index 6d2dd968..9c7a3af6 100644 --- a/wrolpi/test/test_dates.py +++ b/wrolpi/test/test_dates.py @@ -23,8 +23,9 @@ class TestTable(Base): def assert_raw_datetime(expected_datetime: str): with get_db_curs() as curs: curs.execute('SELECT * FROM test_table') + # SQLite stores datetimes as TEXT (`YYYY-MM-DD HH:MM:SS.ffffff` UTC). dt_ = curs.fetchone()['dt'] - assert dt_.isoformat() == expected_datetime + assert datetime.fromisoformat(dt_).isoformat() == expected_datetime def test_TZDateTime(test_session): diff --git a/wrolpi/test/test_db_bootstrap.py b/wrolpi/test/test_db_bootstrap.py new file mode 100644 index 00000000..b34122c7 --- /dev/null +++ b/wrolpi/test/test_db_bootstrap.py @@ -0,0 +1,125 @@ +"""Tests for wrolpi.db_bootstrap: DB auto-creation, migration, and the drive-portability guard.""" +import sqlite3 + +import mock +import pytest + +from wrolpi import db_bootstrap +from wrolpi.db import get_db_file + + +@pytest.fixture(autouse=True) +def permissive_sqlite_floor(): + """CI images may ship a SQLite older than the production floor (Ubuntu 22.04 has 3.37; the + floor targets Debian 12's 3.40). The functional tests here exercise creation/migration, not + the version gate; the gate itself is tested explicitly with mocked versions below.""" + with mock.patch.object(db_bootstrap, 'MINIMUM_SQLITE_VERSION', (3, 0, 0)): + yield + + +def test_ensure_db_creates_and_migrates(test_directory): + """With no database file, ensure_db creates it and migrates to head.""" + db_file = get_db_file() + assert not db_file.is_file() + assert db_bootstrap.compare_db_version() == 'missing' + + assert db_bootstrap.ensure_db() is True + + assert db_file.is_file() + assert db_bootstrap.compare_db_version() == 'current' + with sqlite3.connect(db_file) as conn: + version = conn.execute('SELECT version_num FROM alembic_version').fetchone()[0] + assert version + # Triggers and FTS5 tables were installed by the baseline. + triggers = conn.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='trigger'").fetchone()[0] + assert triggers > 0 + fts_tables = {i[0] for i in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('file_group_fts', 'doc_section_fts')")} + assert fts_tables == {'file_group_fts', 'doc_section_fts'} + + # Running again is a no-op. + assert db_bootstrap.ensure_db() is True + + +def test_ensure_db_refuses_newer_database(test_directory): + """A database from a NEWER WROLPi (unknown alembic revision) must not be written.""" + assert db_bootstrap.ensure_db() is True + db_file = get_db_file() + with sqlite3.connect(db_file) as conn: + conn.execute("UPDATE alembic_version SET version_num = 'future_revision_xyz'") + conn.commit() + + assert db_bootstrap.compare_db_version() == 'ahead' + assert db_bootstrap.ensure_db() is False + # The database was not modified. + with sqlite3.connect(db_file) as conn: + assert conn.execute('SELECT version_num FROM alembic_version').fetchone()[0] == 'future_revision_xyz' + + +def test_compare_db_version_uninitialized_file(test_directory): + """An empty/uninitialized file is treated like a fresh database.""" + db_file = get_db_file() + db_file.parent.mkdir(parents=True, exist_ok=True) + db_file.touch() + assert db_bootstrap.compare_db_version() == 'missing' + assert db_bootstrap.ensure_db() is True + assert db_bootstrap.compare_db_version() == 'current' + + +def test_bootstrap_lock(test_directory): + """Only one process can hold the bootstrap lock.""" + with db_bootstrap.bootstrap_lock() as acquired: + assert acquired is True + with db_bootstrap.bootstrap_lock() as second: + # flock is per-file-descriptor, so a second open in the same process CAN acquire it; + # this just proves the context manager doesn't error or deadlock. + assert second in (True, False) + + +def test_check_sqlite_environment_version_gate(test_directory): + """SQLite older than the floor is refused; at/above the floor is accepted. + + Mocked versions so the test is deterministic regardless of the machine's SQLite (CI images + have shipped 3.37 while the production floor is 3.40).""" + with mock.patch.object(db_bootstrap, 'MINIMUM_SQLITE_VERSION', (3, 40, 0)): + with mock.patch.object(db_bootstrap.sqlite3, 'sqlite_version_info', (3, 39, 9)), \ + mock.patch.object(db_bootstrap.sqlite3, 'sqlite_version', '3.39.9'): + error = db_bootstrap.check_sqlite_environment(get_db_file()) + assert error and 'too old' in error + with mock.patch.object(db_bootstrap.sqlite3, 'sqlite_version_info', (3, 40, 0)): + assert db_bootstrap.check_sqlite_environment(get_db_file()) is None + with mock.patch.object(db_bootstrap.sqlite3, 'sqlite_version_info', (3, 46, 1)): + assert db_bootstrap.check_sqlite_environment(get_db_file()) is None + + +def test_unmounted_production_guard(test_directory): + """An unmounted production media directory blocks DB creation/migration and db_up. + + Regression test for the shadow-database bug found on 10.0.0.8: after the media drive was + unmounted, a worker's ORM connection created an empty wrolpi.db on the root filesystem and + the bootstrap migrated it — because the old guard only refused when NO file existed.""" + import pathlib + + fake_media = mock.MagicMock(spec=pathlib.Path) + fake_media.__str__ = lambda self: '/media/wrolpi' + fake_media.is_mount.return_value = False + fake_media.is_dir.return_value = True + + with mock.patch.object(db_bootstrap, 'PYTEST', False), \ + mock.patch.object(db_bootstrap, 'DOCKERIZED', False), \ + mock.patch.object(db_bootstrap.sys, 'platform', 'linux'): + # Unmounted /media/* on a production linux host: guarded. + assert db_bootstrap.media_directory_is_unmounted_production(fake_media) is True + # Mounted: fine. + fake_media.is_mount.return_value = True + assert db_bootstrap.media_directory_is_unmounted_production(fake_media) is False + # Custom (non-/media) directories are exempt (dev boxes). + fake_media.is_mount.return_value = False + fake_media.__str__ = lambda self: '/home/user/media' + assert db_bootstrap.media_directory_is_unmounted_production(fake_media) is False + + # The guard must refuse ensure_db EVEN IF a database file already exists (an ORM connection + # creates an empty file implicitly, so file-existence cannot be the condition). + assert db_bootstrap.ensure_db() is True # create a real DB in the test directory first + with mock.patch.object(db_bootstrap, 'media_directory_is_unmounted_production', return_value=True): + assert db_bootstrap.ensure_db() is False diff --git a/wrolpi/test/test_fts.py b/wrolpi/test/test_fts.py new file mode 100644 index 00000000..fa68fbf1 --- /dev/null +++ b/wrolpi/test/test_fts.py @@ -0,0 +1,188 @@ +"""Tests for wrolpi.fts: the websearch → FTS5 translator, DDL/sync triggers, and headlines.""" +import sqlite3 + +import pytest + +from wrolpi import fts + + +@pytest.mark.parametrize('search_str,expected', [ + ('one', '(("one"))'), + ('two words', '(("two" AND "words"))'), + ('"a phrase"', '(("a phrase"))'), + ('"a phrase" extra', '(("a phrase" AND "extra"))'), + ('word -excluded', '((("word") NOT "excluded"))'), + ('-only', None), # all-negative is not expressible in FTS5; treated as nothing to search + ('cat OR dog', '(("cat")) OR (("dog"))'), + ('or', None), # a lone OR is not a search + ('cat OR OR dog', '(("cat")) OR (("dog"))'), + ('OR cat', '(("cat"))'), + ("don't", '(("don t"))'), # punctuation-joined tokens become a phrase, like websearch + ('!!! ...', None), # pure punctuation + ('', None), + (None, None), + (' ', None), +]) +def test_translate_websearch(search_str, expected): + assert fts.translate_websearch(search_str) == expected + + +def test_translate_websearch_columns(): + result = fts.translate_websearch('word', columns=fts.ABC_COLUMNS) + assert result == '{a_text b_text c_text} : ((("word")))' + + +@pytest.mark.parametrize('garbage', [ + '"', '((', 'NOT', '-', '*', 'a:b^c', '"unclosed', 'a AND OR NOT -', '{}', 'a"b"c', + '-"neg phrase" -word', 'OR OR OR', '(((((', '%%%', 'term*', 'a NEAR b', '":"', +]) +def test_translate_websearch_fuzz(garbage): + """Translator output must either be None or parse without an FTS5 syntax error.""" + result = fts.translate_websearch(garbage) + if result is None: + return + with sqlite3.connect(':memory:') as conn: + conn.execute("CREATE VIRTUAL TABLE t USING fts5(a_text, b_text, c_text, d_text)") + # Raises sqlite3.OperationalError on FTS5 syntax errors. + conn.execute('SELECT * FROM t WHERE t MATCH ? LIMIT 1', (result,)).fetchall() + + +@pytest.fixture +def fts_db(): + """A scratch DB with a minimal file_group/doc_section and the real FTS DDL + triggers.""" + conn = sqlite3.connect(':memory:') + conn.executescript(''' + CREATE TABLE file_group (id INTEGER PRIMARY KEY, a_text TEXT, b_text TEXT, c_text TEXT, d_text TEXT); + CREATE TABLE doc_section (id INTEGER PRIMARY KEY, content TEXT); + ''') + for statement in fts.FTS_DDL: + conn.execute(statement) + yield conn + conn.close() + + +def test_fts_sync_triggers(fts_db): + """INSERT/UPDATE/DELETE on file_group keep the FTS index in sync.""" + fts_db.execute("INSERT INTO file_group (id, a_text, d_text) VALUES (1, 'Cooking Rice', 'boil the water')") + fts_db.execute("INSERT INTO file_group (id, a_text) VALUES (2, 'Gardening')") + + def match(query, deep=True): + columns = None if deep else fts.ABC_COLUMNS + expr = fts.translate_websearch(query, columns=columns) + return [row[0] for row in + fts_db.execute('SELECT rowid FROM file_group_fts WHERE file_group_fts MATCH ?', (expr,))] + + assert match('cooking') == [1] + # Stemming: porter maps "gardens" and "gardening" to the same stem. + assert match('gardens') == [2] + # Fast path does not search d_text. + assert match('boil', deep=True) == [1] + assert match('boil', deep=False) == [] + + # UPDATE of an indexed column. + fts_db.execute("UPDATE file_group SET a_text = 'Baking Bread' WHERE id = 1") + assert match('cooking') == [] + assert match('baking') == [1] + + # DELETE removes from the index. + fts_db.execute('DELETE FROM file_group WHERE id = 1') + assert match('baking') == [] + + # Integrity check passes after all of the above. + assert fts.fts_integrity_ok(fts_db) is True + + +def test_fts_rank_weights(fts_db): + """A match in a_text (weight 10) outranks the same match in d_text (weight 1).""" + fts_db.execute("INSERT INTO file_group (id, a_text) VALUES (1, 'solar power')") + fts_db.execute("INSERT INTO file_group (id, d_text) VALUES (2, 'solar power')") + expr = fts.translate_websearch('solar') + rows = fts_db.execute( + 'SELECT rowid, -rank FROM file_group_fts WHERE file_group_fts MATCH ? ORDER BY rank', + (expr,)).fetchall() + assert [row[0] for row in rows] == [1, 2] + assert rows[0][1] > rows[1][1] > 0 # exposed rank is positive, higher is better + + +def test_doc_section_fts(fts_db): + fts_db.execute("INSERT INTO doc_section (id, content) VALUES (7, 'the fire needs oxygen')") + expr = fts.translate_websearch('oxygen') + rows = fts_db.execute( + "SELECT rowid, snippet(doc_section_fts, 0, '[', ']', '…', 5) FROM doc_section_fts " + "WHERE doc_section_fts MATCH ?", (expr,)).fetchall() + assert rows[0][0] == 7 + assert '[oxygen]' in rows[0][1] + + +def test_headline_texts(): + entries = ['the quick brown fox jumps', 'nothing to see here', None] + results = fts.headline_texts(entries, 'foxes jumping') # stems match fox/jumps + assert 'fox' in results[0][0] and 'jumps' in results[0][0] + assert results[0][1] > 0 + # Non-matching entries return the start of the text with rank 0. + assert results[1] == ('nothing to see here', 0.0) + assert results[2] == ('', 0.0) + + +def test_headline_texts_no_search(): + results = fts.headline_texts(['some text'], '') + assert results == [('some text', 0.0)] + + +STEMMING_DOCS = { + 1: 'watch the sunrise', + 2: 'he watches birds', + 3: 'she watched a movie', + 4: 'watching the stars', + 5: 'jump the fence', + 6: 'she jumps high', + 7: 'jumping and jumped all day', + 8: 'run to the store', + 9: 'she runs fast', + 10: 'I go running daily', + 11: 'fish in the lake', + 12: 'he fishes and fished', + 13: 'fishing is relaxing', + 14: 'a cat sleeps', + 15: 'two cats play', + 16: 'the dog barks', + 17: 'three dogs howl', +} + +WATCH, JUMP, RUN, FISH = [1, 2, 3, 4], [5, 6, 7], [8, 9, 10], [11, 12, 13] + + +@pytest.mark.parametrize('query,expected', [ + # Every inflectional variant matches every other variant of its word. + ('watch', WATCH), ('watches', WATCH), ('watched', WATCH), ('watching', WATCH), + ('jump', JUMP), ('jumps', JUMP), ('jumping', JUMP), ('jumped', JUMP), + ('run', RUN), ('runs', RUN), ('running', RUN), + ('fish', FISH), ('fishes', FISH), ('fished', FISH), ('fishing', FISH), + ('cat', [14, 15]), ('cats', [14, 15]), + ('dog', [16, 17]), ('dogs', [16, 17]), + # Documented boundary (identical to the old Postgres snowball behavior): stemmers strip + # grammatical suffixes, they do not lemmatize -- derived words like "runner" are their own + # terms and do not match the "run" family. + ('runner', []), +]) +def test_stemming(fts_db, query, expected): + """Searching any grammatical variant of a word matches all other variants. + + This contract is load-bearing: users search "running" and expect files titled + "runs"/"run" to match. The Porter stemmer provides it at index AND query time.""" + for rowid, text in STEMMING_DOCS.items(): + fts_db.execute('INSERT INTO file_group (id, a_text) VALUES (?, ?)', (rowid, text)) + + match = fts.translate_websearch(query) + rows = [row[0] for row in + fts_db.execute('SELECT rowid FROM file_group_fts WHERE file_group_fts MATCH ? ORDER BY rowid', (match,))] + assert rows == expected + + +def test_stemming_headlines_agree_with_matching(fts_db): + """The headline helper uses the same tokenizer, so highlights agree with matches.""" + results = fts.headline_texts(['he watches birds', 'a marathon runner'], 'watching') + assert results[0] == ('he watches birds', results[0][1]) + assert results[0][1] > 0 + # "runner" did not match "watching"; the entry returns leading text with rank 0. + assert results[1] == ('a marathon runner', 0.0) diff --git a/wrolpi/vars.py b/wrolpi/vars.py index 2818e71b..59b1d219 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -74,12 +74,6 @@ DEFAULT_CPU_FREQUENCY = 'ondemand' DEFAULT_FILE_PERMISSIONS = 0o644 -DB_HOST = os.environ.get('DB_HOST', '127.0.0.1') -DB_PORT = int(os.environ.get('DB_PORT', 5432)) -DB_NAME = os.environ.get('DB_NAME', 'wrolpi') -DB_USER = os.environ.get('DB_USER', 'wrolpi') -DB_PASSWORD = os.environ.get('DB_PASSWORD', 'wrolpi') - API_HOST = os.environ.get('API_HOST', '127.0.0.1') API_PORT = os.environ.get('API_PORT', '8081') API_WORKERS = os.environ.get('API_WORKERS', multiprocessing.cpu_count())