mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Replacing tag filtering with new tag_names_to_sub_select function which uses a single sub-select to filter FileGroups.
This commit is contained in:
parent
fedbbe31d7
commit
bc2ca0e512
5 changed files with 26 additions and 61 deletions
|
|
@ -20,7 +20,7 @@ from wrolpi.common import get_media_directory, logger, extract_domain, chdir, es
|
|||
from wrolpi.dates import now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs, optional_session
|
||||
from wrolpi.errors import UnknownArchive, InvalidOrderBy
|
||||
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_clauses
|
||||
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_sub_select
|
||||
from wrolpi.vars import DOCKERIZED, PYTEST
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -416,10 +416,10 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
|
|||
raise InvalidOrderBy(f'Invalid order by: {order}')
|
||||
|
||||
if tag_names:
|
||||
where_, params_, join_ = tag_names_to_clauses(tag_names)
|
||||
wheres.append(where_)
|
||||
# Filter all FileGroups by those that have been tagged with the provided tag names.
|
||||
tags_stmt, params_ = tag_names_to_sub_select(tag_names)
|
||||
params.update(params_)
|
||||
joins.append(join_)
|
||||
wheres.append(f'fg.id = ANY({tags_stmt})')
|
||||
|
||||
if search_str and headline:
|
||||
headline = ''',
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from modules.videos.models import Video
|
|||
from wrolpi.common import run_after, logger
|
||||
from wrolpi.db import get_db_session, optional_session
|
||||
from wrolpi.errors import UnknownVideo
|
||||
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_clauses
|
||||
from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_sub_select
|
||||
from ..lib import save_channels_config
|
||||
|
||||
logger.getChild(__name__)
|
||||
|
|
@ -102,10 +102,10 @@ def search_videos(
|
|||
select_columns = 'fg.id, COUNT(*) OVER() AS total'
|
||||
|
||||
if tag_names:
|
||||
where_, params_, join_ = tag_names_to_clauses(tag_names)
|
||||
wheres.append(where_)
|
||||
# Filter all FileGroups by those that have been tagged with the provided tag names.
|
||||
tags_stmt, params_ = tag_names_to_sub_select(tag_names)
|
||||
params.update(params_)
|
||||
joins.append(join_)
|
||||
wheres.append(f'fg.id = ANY({tags_stmt})')
|
||||
|
||||
if search_str and headline:
|
||||
headline = ''',
|
||||
|
|
|
|||
|
|
@ -648,10 +648,10 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
|
|||
wheres.append('model = %(model)s')
|
||||
|
||||
if tags:
|
||||
where_, params_, join_ = tag_names_to_clauses(tags)
|
||||
wheres.append(where_)
|
||||
# Filter all FileGroups by those that have been tagged with the provided tag names.
|
||||
tags_stmt, params_ = tag_names_to_sub_select(tags)
|
||||
params.update(params_)
|
||||
joins.append(join_)
|
||||
wheres.append(f'fg.id = ANY({tags_stmt})')
|
||||
|
||||
if search_str and headline:
|
||||
headline = ''',
|
||||
|
|
@ -681,24 +681,22 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
|
|||
return results, total
|
||||
|
||||
|
||||
def tag_names_to_clauses(tags: List[str]):
|
||||
"""Create the SQL necessary to filter the `file_group` table by the provided Tag names."""
|
||||
params = dict()
|
||||
def tag_names_to_sub_select(names: List[str]) -> Tuple[str, dict]:
|
||||
"""Create the SQL necessary to filter FileGroup by the provided Tag names."""
|
||||
if not names:
|
||||
return '', dict()
|
||||
|
||||
if not tags:
|
||||
return '', params, ''
|
||||
|
||||
where_tags = []
|
||||
joins = []
|
||||
for idx, tag_name in enumerate(tags):
|
||||
where_tags.append(f't{idx}.name = %(tag_name{idx})s')
|
||||
params[f'tag_name{idx}'] = tag_name
|
||||
joins.append(f'LEFT JOIN tag_file tf{idx} ON tf{idx}.file_group_id = fg.id '
|
||||
f'LEFT JOIN tag t{idx} ON t{idx}.id = tf{idx}.tag_id')
|
||||
where_tags = ' AND '.join(where_tags)
|
||||
wheres = f'({where_tags})'
|
||||
join = '\n'.join(joins)
|
||||
return wheres, params, join
|
||||
stmt = '''
|
||||
SELECT
|
||||
tf.file_group_id
|
||||
FROM
|
||||
tag_file tf
|
||||
LEFT JOIN tag t on t.id = tf.tag_id
|
||||
GROUP BY file_group_id
|
||||
-- Match only FileGroups that have at least all the Tag names.
|
||||
HAVING array_agg(t.name) @> %(tag_names)s::VARCHAR[]
|
||||
'''
|
||||
return stmt, dict(tag_names=names)
|
||||
|
||||
|
||||
def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List[dict], int]:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import asyncio
|
||||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
|
|
@ -11,7 +10,6 @@ from uuid import uuid4
|
|||
|
||||
import mock
|
||||
import pytest
|
||||
import sqlalchemy.exc
|
||||
from PIL import Image
|
||||
|
||||
from wrolpi.common import timer
|
||||
|
|
@ -543,36 +541,6 @@ def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, e
|
|||
assert lib.get_primary_file([example_pdf, example_mobi, example_epub]) == example_epub
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'tags,wheres,params,join', [
|
||||
(
|
||||
['one'],
|
||||
'(t0.name = %(tag_name0)s)',
|
||||
dict(tag_name0='one'),
|
||||
'LEFT JOIN tag_file tf0 ON tf0.file_group_id = fg.id LEFT JOIN tag t0 ON t0.id = tf0.tag_id',
|
||||
),
|
||||
(
|
||||
['one', 'two'],
|
||||
'(t0.name = %(tag_name0)s AND t1.name = %(tag_name1)s)',
|
||||
dict(tag_name0='one', tag_name1='two'),
|
||||
'LEFT JOIN tag_file tf0 ON tf0.file_group_id = fg.id LEFT JOIN tag t0 ON t0.id = tf0.tag_id'
|
||||
'\nLEFT JOIN tag_file tf1 ON tf1.file_group_id = fg.id LEFT JOIN tag t1 ON t1.id = tf1.tag_id',
|
||||
),
|
||||
(
|
||||
[],
|
||||
'',
|
||||
dict(),
|
||||
'',
|
||||
),
|
||||
]
|
||||
)
|
||||
def test_tag_names_to_clauses(tags, wheres, params, join):
|
||||
result = lib.tag_names_to_clauses(tags)
|
||||
assert result[0] == wheres
|
||||
assert result[1] == params
|
||||
assert result[2] == join
|
||||
|
||||
|
||||
def test_get_refresh_progress(test_client, test_session):
|
||||
request, response = test_client.get('/api/files/refresh_progress')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import pytest
|
|||
import yaml
|
||||
|
||||
from wrolpi import tags
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.errors import FileGroupIsTagged
|
||||
from wrolpi.files import lib as files_lib
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue