rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-25 18:40:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Running alembic check in cicd.

Browse source
This commit is contained in:
Roland 2025-11-29 09:14:32 -07:00 • committed by lrnselfreliance
parent a1e0fb4e92
commit 1d8738ace2
15 changed files with 174 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.circleci/config.yml
View file

@ -58,7 +58,11 @@ jobs:
paths:
- "venv"
- run:
name: Run Alembic Migrations
command: ./venv/bin/python ./main.py db upgrade
- run:
name: Check for Missing Migrations
command: ./venv/bin/python ./main.py db check
api-tests-3-12:
docker:
- image: cimg/python:3.12

17
alembic/env.py
View file

@ -15,9 +15,20 @@ fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None
from wrolpi.common import Base
# Import all models so they are registered with the Base metadata
from wrolpi.files.models import FileGroup, Directory # noqa: F401
from wrolpi.files.ebooks import EBook # noqa: F401
from wrolpi.tags import Tag # noqa: F401
from wrolpi.flags import WROLPiFlag # noqa: F401
from wrolpi.downloader import Download # noqa: F401
from wrolpi.collections.models import Collection, CollectionItem # noqa: F401
from modules.videos.models import Video, Channel # noqa: F401
from modules.archive.models import Archive # noqa: F401
from modules.zim.models import Zim, TagZimEntry, ZimSubscription # noqa: F401
from modules.map.models import MapFile # noqa: F401
from modules.inventory.models import Item, Inventory # noqa: F401
target_metadata = Base.metadata
# other values from the config, defined by the needs of env.py,

68
alembic/versions/f38cfe4b5cb2_collections_feature.py Normal file
View file

@ -0,0 +1,68 @@
"""collections feature
Revision ID: f38cfe4b5cb2
Revises: add_unique_collection_name_kind
Create Date: 2025-11-29 16:51:34.222412
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = 'f38cfe4b5cb2'
down_revision = 'add_unique_collection_name_kind'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('archive', 'collection_id',
existing_type=sa.INTEGER(),
nullable=True)
op.alter_column('file_group', 'mimetype',
existing_type=sa.TEXT(),
nullable=True)
op.alter_column('tag_file', 'tag_id',
existing_type=sa.INTEGER(),
nullable=False)
op.alter_column('tag_file', 'file_group_id',
existing_type=sa.BIGINT(),
nullable=False)
op.alter_column('tag_zim', 'tag_id',
existing_type=sa.INTEGER(),
nullable=False)
op.alter_column('tag_zim', 'zim_id',
existing_type=sa.INTEGER(),
nullable=False)
op.drop_index('video_upload_date_idx', table_name='video')
op.drop_column('video', 'duration')
op.drop_column('video', 'upload_date')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('video', sa.Column('upload_date', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True))
op.add_column('video', sa.Column('duration', sa.INTEGER(), autoincrement=False, nullable=True))
op.create_index('video_upload_date_idx', 'video', ['upload_date'], unique=False)
op.alter_column('tag_zim', 'zim_id',
existing_type=sa.INTEGER(),
nullable=True)
op.alter_column('tag_zim', 'tag_id',
existing_type=sa.INTEGER(),
nullable=True)
op.alter_column('tag_file', 'file_group_id',
existing_type=sa.BIGINT(),
nullable=True)
op.alter_column('tag_file', 'tag_id',
existing_type=sa.INTEGER(),
nullable=True)
op.alter_column('archive', 'collection_id',
existing_type=sa.INTEGER(),
nullable=False)
op.alter_column('file_group', 'mimetype',
existing_type=sa.TEXT(),
nullable=False)
# ### end Alembic commands ###

10
main.py
View file

@ -32,7 +32,7 @@ logger = logger.getChild('wrolpi-main')
def db_main(args):
"""
Handle database migrations. This uses Alembic, supported commands are "upgrade" and "downgrade".
Handle database migrations. This uses Alembic, supported commands are "upgrade", "downgrade", and "check".
"""
from alembic.config import Config
from alembic import command
@ -48,6 +48,11 @@ def db_main(args):
command.upgrade(config, 'head')
elif args.command == 'downgrade':
command.downgrade(config, '-1')
elif args.command == 'check':
command.check(config)
elif args.command == 'revision':
message = getattr(args, 'message', None) or 'auto migration'
command.revision(config, message=message, autogenerate=True)
else:
print(f'Unknown DB command: {args.command}')
return 2
@ -100,7 +105,8 @@ def main():
# DB Parser for running Alembic migrations
db_parser = sub_commands.add_parser('db')
db_parser.add_argument('command', help='Supported commands: upgrade, downgrade')
db_parser.add_argument('command', help='Supported commands: upgrade, downgrade, check, revision')
db_parser.add_argument('-m', '--message', help='Migration message (for revision command)')
args = parser.parse_args()

9
modules/archive/__init__.py
View file

@ -190,6 +190,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
# Set collection_id BEFORE adding to session to avoid autoflush constraint violation
from modules.archive.lib import get_or_create_domain_collection
if file_group.url:
# URL is already set on the FileGroup
collection = get_or_create_domain_collection(session, file_group.url)
@ -202,12 +203,8 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
collection = get_or_create_domain_collection(session, url)
archive.collection_id = collection.id if collection else None
except (RuntimeError, ValueError) as e:
# Could not extract URL from singlefile
if not PYTEST:
# In production, archives must have a URL/collection
raise InvalidArchive(f'Archive has no URL and could not extract from singlefile: {e}') from e
# In tests, allow archives without collections (factory will set it later)
logger.debug(f'Could not extract URL from singlefile (test mode): {e}')
# Could not extract URL from singlefile - archive will have no collection
logger.debug(f'Could not extract URL from singlefile: {e}')
archive.collection_id = None
session.add(archive)

7
modules/inventory/models.py
View file

@ -23,7 +23,7 @@ class Item(Base, ModelHelper):
subcategory = Column(String)
unit = Column(String)
inventory_id = Column(Integer, ForeignKey('inventory.id'))
inventory_id = Column(Integer, ForeignKey('inventory.id', ondelete='CASCADE'))
inventory = relationship('Inventory', primaryjoin="Item.inventory_id==Inventory.id", back_populates='items')
def __repr__(self):
@ -35,7 +35,7 @@ class Inventory(Base, ModelHelper):
__tablename__ = 'inventory'
id = Column(Integer, primary_key=True)
name = Column(String, unique=True)
name = Column(String, unique=True, nullable=False)
viewed_at = Column(TZDateTime)
created_at = Column(TZDateTime, default=now)
deleted_at = Column(TZDateTime)
@ -61,6 +61,3 @@ class Inventory(Base, ModelHelper):
return session.query(Inventory).filter_by(name=name).one()
class InventoriesVersion(Base, ModelHelper):
__tablename__ = 'inventories_version'
version = Column(Integer, primary_key=True)

4
modules/map/models.py
View file

@ -10,8 +10,8 @@ class MapFile(Base, ModelHelper):
__tablename__ = 'map_file'
id = Column(Integer, primary_key=True)
path = Column(MediaPathType, unique=True, nullable=False)
imported = Column(Boolean, default=False, nullable=False)
path = Column(MediaPathType, nullable=False)
imported = Column(Boolean, default=False)
size = Column(BigInteger)
def __repr__(self):

20
modules/videos/models.py
View file

@ -2,7 +2,7 @@ import json
import pathlib
from typing import Optional, Dict, List
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ForeignKey, BigInteger
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ForeignKey, BigInteger, Index, text
from sqlalchemy.orm import relationship, Session, deferred
from sqlalchemy.orm.collections import InstrumentedList
@ -28,6 +28,11 @@ __all__ = ['Video', 'Channel']
class Video(ModelHelper, Base):
__tablename__ = 'video'
__table_args__ = (
Index('video_channel_id_idx', 'channel_id'),
Index('video_source_id_idx', 'source_id'),
Index('video_view_count_idx', 'view_count'),
)
id = Column(Integer, primary_key=True)
source_id = Column(String) # The id from yt-dlp
@ -530,9 +535,16 @@ class Video(ModelHelper, Base):
class Channel(ModelHelper, Base):
__tablename__ = 'channel'
__table_args__ = (
Index('channel_minimum_frequency_idx', 'minimum_frequency'),
Index('channel_source_id', 'source_id'),
Index('channel_total_size_idx', 'total_size'),
Index('channel_video_count_idx', 'video_count'),
Index('channel_url_key', 'url', unique=True, postgresql_where=text('url IS NOT NULL')),
)
id = Column(Integer, primary_key=True)
# name and directory are stored in Collection, accessed via properties
url = Column(String, unique=True) # will only be downloaded if related Download exists.
url = Column(String) # will only be downloaded if related Download exists. Partial unique index on non-NULL values.
generate_posters = Column(Boolean, default=False) # generating posters may delete files, and can be slow.
calculate_duration = Column(Boolean, default=True) # use ffmpeg to extract duration (slower than info json).
download_missing_data = Column(Boolean, default=True) # fetch missing data like `source_id` and video comments.
@ -540,8 +552,8 @@ class Channel(ModelHelper, Base):
refreshed = Column(Boolean, default=False) # The files in the Channel have been refreshed.
# Columns updated by triggers
video_count = Column(Integer, default=0) # update_channel_video_count
total_size = Column(Integer, default=0) # update_channel_size
video_count = Column(Integer, default=0, nullable=False) # update_channel_video_count
total_size = Column(BigInteger, default=0, nullable=False) # update_channel_size
minimum_frequency = Column(Integer) # update_channel_minimum_frequency
info_json = deferred(Column(JSON))

14
modules/zim/models.py
View file

@ -6,7 +6,7 @@ from datetime import datetime
from typing import List, Tuple, OrderedDict as OrderedDictType, Dict, Optional, Set
from libzim import Archive, Searcher, Query, Entry, SuggestionSearcher
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean, UniqueConstraint
from sqlalchemy.orm import relationship, Session
from sqlalchemy.orm.exc import NoResultFound # noqa
@ -67,12 +67,15 @@ class Zim(Base, ModelHelper):
"""
__tablename__ = 'zim'
__table_args__ = (
UniqueConstraint('path', name='zim_path_key'),
)
id = Column(Integer, primary_key=True)
path: pathlib.Path = Column(MediaPathType, nullable=False)
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False)
file_group: FileGroup = relationship('FileGroup')
auto_search = Column(Boolean, nullable=False, default=True)
auto_search = Column(Boolean, default=True)
def __repr__(self):
return f'<Zim id={self.id} file_group_id={self.file_group_id} path={self.path}>'
@ -353,8 +356,11 @@ class Zims:
class TagZimEntry(Base):
__tablename__ = 'tag_zim'
__table_args__ = (
UniqueConstraint('tag_id', 'zim_id', 'zim_entry', name='tag_zim_tag_id_zim_id_zim_entry_key'),
)
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True)
tag_id = Column(Integer, ForeignKey('tag.id'), primary_key=True)
tag: Tag = relationship('Tag')
zim_id = Column(Integer, ForeignKey('zim.id', ondelete='CASCADE'), primary_key=True)
zim: Zim = relationship('Zim')

5
modules/zim/test/test_lib.py
View file

@ -161,10 +161,9 @@ async def test_zim_tags_config(async_client, test_session, test_directory, test_
['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
]
# Delete all Tags so they are recreated.
test_session.query(tags.Tag).delete()
# Delete all TagFileEntry(s), import them again.
# Delete all TagZimEntry(s) first (due to FK constraint), then Tags so they are recreated.
test_session.query(TagZimEntry).delete()
test_session.query(tags.Tag).delete()
test_session.commit()
tags.import_tags_config()

18
wrolpi/collections/models.py
View file

@ -1,7 +1,7 @@
import pathlib
from typing import Optional, List
from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Index, UniqueConstraint, func
from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Index, UniqueConstraint, func, BigInteger
from sqlalchemy.orm import relationship, Session
from sqlalchemy.orm.collections import InstrumentedList
@ -67,6 +67,8 @@ class Collection(ModelHelper, Base):
UniqueConstraint('directory', name='uq_collection_directory'),
UniqueConstraint('name', 'kind', name='uq_collection_name_kind'),
Index('idx_collection_kind', 'kind'),
Index('idx_collection_item_count', 'item_count'),
Index('idx_collection_total_size', 'total_size'),
)
id = Column(Integer, primary_key=True)
@ -80,14 +82,14 @@ class Collection(ModelHelper, Base):
directory: Optional[pathlib.Path] = Column(MediaPathType, nullable=True)
# Optional tag relationship (similar to Channel)
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'))
tag_id = Column(Integer, ForeignKey('tag.id'))
tag = relationship('Tag', primaryjoin='Collection.tag_id==Tag.id')
created_date = Column(DateTime, server_default=func.now())
created_date = Column(DateTime, server_default=func.now(), nullable=False)
# Columns updated by triggers (similar to Channel)
item_count = Column(Integer, default=0)
total_size = Column(Integer, default=0)
item_count = Column(Integer, default=0, nullable=False)
total_size = Column(BigInteger, default=0, nullable=False)
# Relationship to items (ordered)
items: InstrumentedList = relationship(
@ -790,7 +792,7 @@ class CollectionItem(ModelHelper, Base):
position = Column(Integer, nullable=False, default=0)
# When this item was added
added_date = Column(DateTime, server_default=func.now())
added_date = Column(DateTime, server_default=func.now(), nullable=False)
# Relationships
collection = relationship('Collection', back_populates='items')
@ -798,8 +800,10 @@ class CollectionItem(ModelHelper, Base):
# Indexes for performance
__table_args__ = (
Index('idx_collection_item_collection_id', 'collection_id'),
Index('idx_collection_item_collection_position', 'collection_id', 'position'),
Index('idx_collection_item_file_group', 'file_group_id'),
Index('idx_collection_item_file_group_id', 'file_group_id'),
Index('idx_collection_item_position', 'position'),
UniqueConstraint('collection_id', 'file_group_id', name='uq_collection_file_group'),
)

5
wrolpi/downloader.py
View file

@ -20,7 +20,7 @@ from urllib.parse import urlparse
import feedparser
import pytz
from feedparser import FeedParserDict
from sqlalchemy import Column, Integer, String, Text, ForeignKey, ARRAY
from sqlalchemy import Column, Integer, String, Text, ForeignKey, ARRAY, Index
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Session, relationship
from sqlalchemy.sql import Delete
@ -100,6 +100,9 @@ class DownloadStatus(str, Enum):
class Download(ModelHelper, Base): # noqa
"""Model that is used to schedule downloads."""
__tablename__ = 'download' # noqa
__table_args__ = (
Index('idx_download_collection_id', 'collection_id'),
)
id = Column(Integer, primary_key=True)
url = Column(String, nullable=False, unique=True)

5
wrolpi/files/ebooks.py
View file

@ -4,7 +4,7 @@ import logging
import pathlib
from typing import List, Optional, Tuple
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, or_, and_
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, or_, and_, Index
from sqlalchemy.orm import relationship, Session
from wrolpi.common import ModelHelper, Base, register_modeler, get_html_soup
@ -150,6 +150,9 @@ def discover_calibre_cover(ebook_path: pathlib.Path):
class EBook(ModelHelper, Base):
__tablename__ = 'ebook'
__table_args__ = (
Index('ebook_size_idx', 'size'),
)
id = Column(Integer, primary_key=True)
size: int = Column(Integer)

24
wrolpi/files/models.py
View file

@ -6,7 +6,7 @@ from datetime import datetime
from functools import singledispatchmethod
from typing import List, Type, Optional, Iterable
from sqlalchemy import Column, String, Computed, BigInteger, Boolean, event
from sqlalchemy import Column, String, Computed, BigInteger, Boolean, event, Index
from sqlalchemy import types
from sqlalchemy.orm import deferred, relationship, Session
@ -66,15 +66,30 @@ class FancyJSON(types.TypeDecorator):
class FileGroup(ModelHelper, Base):
__tablename__ = 'file_group'
__table_args__ = (
Index('file_group_author_idx', 'author'),
Index('file_group_download_datetime_idx', 'download_datetime'),
Index('file_group_effective_datetime_idx', 'effective_datetime'),
Index('file_group_length_idx', 'length'),
Index('file_group_mimetype_idx', 'mimetype'),
Index('file_group_model_idx', 'model'),
Index('file_group_modification_datetime_idx', 'modification_datetime'),
Index('file_group_published_datetime_idx', 'published_datetime'),
Index('file_group_published_modified_datetime_idx', 'published_modified_datetime'),
Index('file_group_size_ix', 'size'),
Index('file_group_textsearch_idx', 'textsearch'),
Index('file_group_url_idx', 'url'),
Index('file_group_viewed_idx', 'viewed'),
)
id: int = Column(BigInteger, primary_key=True)
author = Column(String) # name of the author, maybe even a URL
censored = Column(Boolean) # the file is no longer available for download
data = Column(FancyJSON) # populated by the modeler
download_datetime = Column(TZDateTime) # the date WROLPi downloaded this file.
files = Column(FancyJSON, nullable=False) # populated during discovery
files = Column(FancyJSON) # populated during discovery
idempotency = Column(TZDateTime) # used to track which files need to be deleted during refresh
indexed = Column(Boolean, default=lambda: False, nullable=False) # wrolpi.files.lib.apply_indexers
indexed = Column(Boolean, default=lambda: False) # wrolpi.files.lib.apply_indexers
length = Column(BigInteger) # video duration, article words, etc.
mimetype = Column(String) # wrolpi.files.lib.get_mimetype
model = Column(String) # "video", "archive", "ebook", etc.
@ -550,6 +565,9 @@ def update_effective_datetime(mapper, connection, target):
class Directory(ModelHelper, Base):
"""A representation of a file directory in the media directory."""
__tablename__ = 'directory'
__table_args__ = (
Index('directory_name_idx', 'name'),
)
path: pathlib.Path = Column(MediaPathType, primary_key=True)
name: str = Column(String, nullable=False)

7
wrolpi/tags.py
View file

@ -6,7 +6,7 @@ from typing import List, Dict, Tuple, Optional
import cachetools
from cachetools.keys import hashkey
from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger, event
from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger, event, UniqueConstraint
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import relationship, Session
@ -30,9 +30,12 @@ def get_tags_directory() -> pathlib.Path:
class TagFile(ModelHelper, Base):
__tablename__ = 'tag_file'
__table_args__ = (
UniqueConstraint('tag_id', 'file_group_id', name='tag_file_tag_id_file_group_id_key'),
)
created_at: datetime = Column(TZDateTime, default=dates.now)
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True)
tag_id = Column(Integer, ForeignKey('tag.id'), primary_key=True)
tag = relationship('Tag', back_populates='tag_files')
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), primary_key=True)
file_group = relationship('FileGroup', back_populates='tag_files')