diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 24fa18f9..36cc6c4d 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -20,7 +20,7 @@ from wrolpi.common import get_media_directory, logger, extract_domain, chdir, es from wrolpi.dates import now, Seconds from wrolpi.db import get_db_session, get_db_curs, optional_session from wrolpi.errors import UnknownArchive, InvalidOrderBy -from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_clauses +from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_sub_select from wrolpi.vars import DOCKERIZED, PYTEST logger = logger.getChild(__name__) @@ -416,10 +416,10 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order raise InvalidOrderBy(f'Invalid order by: {order}') if tag_names: - where_, params_, join_ = tag_names_to_clauses(tag_names) - wheres.append(where_) + # Filter all FileGroups by those that have been tagged with the provided tag names. + tags_stmt, params_ = tag_names_to_sub_select(tag_names) params.update(params_) - joins.append(join_) + wheres.append(f'fg.id = ANY({tags_stmt})') if search_str and headline: headline = ''', diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index a3760bde..53099f6a 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -7,7 +7,7 @@ from modules.videos.models import Video from wrolpi.common import run_after, logger from wrolpi.db import get_db_session, optional_session from wrolpi.errors import UnknownVideo -from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_clauses +from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_sub_select from ..lib import save_channels_config logger.getChild(__name__) @@ -102,10 +102,10 @@ def search_videos( select_columns = 'fg.id, COUNT(*) OVER() AS total' if tag_names: - where_, params_, join_ = tag_names_to_clauses(tag_names) - wheres.append(where_) + # Filter all FileGroups by those that have been tagged with the provided tag names. + tags_stmt, params_ = tag_names_to_sub_select(tag_names) params.update(params_) - joins.append(join_) + wheres.append(f'fg.id = ANY({tags_stmt})') if search_str and headline: headline = ''', diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 07e05d23..a3afc7f2 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -648,10 +648,10 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] wheres.append('model = %(model)s') if tags: - where_, params_, join_ = tag_names_to_clauses(tags) - wheres.append(where_) + # Filter all FileGroups by those that have been tagged with the provided tag names. + tags_stmt, params_ = tag_names_to_sub_select(tags) params.update(params_) - joins.append(join_) + wheres.append(f'fg.id = ANY({tags_stmt})') if search_str and headline: headline = ''', @@ -681,24 +681,22 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str] return results, total -def tag_names_to_clauses(tags: List[str]): - """Create the SQL necessary to filter the `file_group` table by the provided Tag names.""" - params = dict() +def tag_names_to_sub_select(names: List[str]) -> Tuple[str, dict]: + """Create the SQL necessary to filter FileGroup by the provided Tag names.""" + if not names: + return '', dict() - if not tags: - return '', params, '' - - where_tags = [] - joins = [] - for idx, tag_name in enumerate(tags): - where_tags.append(f't{idx}.name = %(tag_name{idx})s') - params[f'tag_name{idx}'] = tag_name - joins.append(f'LEFT JOIN tag_file tf{idx} ON tf{idx}.file_group_id = fg.id ' - f'LEFT JOIN tag t{idx} ON t{idx}.id = tf{idx}.tag_id') - where_tags = ' AND '.join(where_tags) - wheres = f'({where_tags})' - join = '\n'.join(joins) - return wheres, params, join + stmt = ''' + SELECT + tf.file_group_id + FROM + tag_file tf + LEFT JOIN tag t on t.id = tf.tag_id + GROUP BY file_group_id + -- Match only FileGroups that have at least all the Tag names. + HAVING array_agg(t.name) @> %(tag_names)s::VARCHAR[] + ''' + return stmt, dict(tag_names=names) def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List[dict], int]: diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 6433e2cc..050c39e8 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -1,6 +1,5 @@ import asyncio import json -import pathlib import shutil import zipfile from datetime import datetime @@ -11,7 +10,6 @@ from uuid import uuid4 import mock import pytest -import sqlalchemy.exc from PIL import Image from wrolpi.common import timer @@ -543,36 +541,6 @@ def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, e assert lib.get_primary_file([example_pdf, example_mobi, example_epub]) == example_epub -@pytest.mark.parametrize( - 'tags,wheres,params,join', [ - ( - ['one'], - '(t0.name = %(tag_name0)s)', - dict(tag_name0='one'), - 'LEFT JOIN tag_file tf0 ON tf0.file_group_id = fg.id LEFT JOIN tag t0 ON t0.id = tf0.tag_id', - ), - ( - ['one', 'two'], - '(t0.name = %(tag_name0)s AND t1.name = %(tag_name1)s)', - dict(tag_name0='one', tag_name1='two'), - 'LEFT JOIN tag_file tf0 ON tf0.file_group_id = fg.id LEFT JOIN tag t0 ON t0.id = tf0.tag_id' - '\nLEFT JOIN tag_file tf1 ON tf1.file_group_id = fg.id LEFT JOIN tag t1 ON t1.id = tf1.tag_id', - ), - ( - [], - '', - dict(), - '', - ), - ] -) -def test_tag_names_to_clauses(tags, wheres, params, join): - result = lib.tag_names_to_clauses(tags) - assert result[0] == wheres - assert result[1] == params - assert result[2] == join - - def test_get_refresh_progress(test_client, test_session): request, response = test_client.get('/api/files/refresh_progress') assert response.status_code == HTTPStatus.OK diff --git a/wrolpi/test/test_tags.py b/wrolpi/test/test_tags.py index ac49b25e..624f9aee 100644 --- a/wrolpi/test/test_tags.py +++ b/wrolpi/test/test_tags.py @@ -5,7 +5,6 @@ import pytest import yaml from wrolpi import tags -from wrolpi.dates import now from wrolpi.errors import FileGroupIsTagged from wrolpi.files import lib as files_lib from wrolpi.files.models import FileGroup