From cd2ce1d69160bb09e40d3b91e7ccf5d2adb70763 Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 1 Feb 2022 09:56:59 -0700 Subject: [PATCH 1/9] Testing bad captions file parsing. --- modules/videos/captions.py | 20 +++++--------------- modules/videos/lib.py | 4 ++-- modules/videos/test/test_captions.py | 15 +++++++++++++-- requirements.txt | 2 +- test/bad_caption.en.vtt | 8 ++++++++ 5 files changed, 29 insertions(+), 20 deletions(-) create mode 100644 test/bad_caption.en.vtt diff --git a/modules/videos/captions.py b/modules/videos/captions.py index 5b60d83b..8f098c1a 100755 --- a/modules/videos/captions.py +++ b/modules/videos/captions.py @@ -1,5 +1,4 @@ #! /usr/bin/env python3 -import pathlib from pathlib import Path from typing import Generator, Union @@ -7,14 +6,12 @@ import srt import webvtt from wrolpi.common import logger -from wrolpi.media_path import MediaPath -from .models import Video + +__all__ = ['get_captions'] def get_caption_text(caption_path: Union[str, Path]) -> Generator: - """ - Return all text from each caption of a caption file. - """ + """Return all text from each caption of a caption file.""" if str(caption_path).endswith('vtt'): for caption in webvtt.read(caption_path): text = str(caption.text).strip() @@ -38,15 +35,8 @@ def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator: yield line -def get_video_captions(video: Video): - """ - Parse video captions from the video's captions file. - """ - if isinstance(video.caption_path, MediaPath): - caption_path = video.caption_path.path - else: - caption_path = pathlib.Path(video.caption_path) - +def get_captions(caption_path: Path): + """Parse video captions from the video's captions file.""" try: lines = get_unique_caption_lines(str(caption_path)) block = '\n'.join(lines) diff --git a/modules/videos/lib.py b/modules/videos/lib.py index f6610a84..f6cb4fab 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -14,7 +14,7 @@ from wrolpi.dates import from_timestamp, Seconds from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.media_path import MediaPath from wrolpi.vars import PYTEST -from .captions import get_video_captions +from .captions import get_captions from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, import_videos_config, \ get_video_duration, is_valid_poster, convert_image, generate_video_poster from .models import Channel, Video @@ -188,7 +188,7 @@ def validate_video(video: Video, channel_generate_poster: bool): # Video duration was not in the info json, use ffprobe. video.duration = get_video_duration(video_path) if not video.caption and video.caption_path: - video.caption = get_video_captions(video) + video.caption = get_captions(video.caption_path.path) if not video.size or not video.modification_datetime: stat = video_path.stat() diff --git a/modules/videos/test/test_captions.py b/modules/videos/test/test_captions.py index 894e80d1..3dbc4cdb 100644 --- a/modules/videos/test/test_captions.py +++ b/modules/videos/test/test_captions.py @@ -1,3 +1,4 @@ +import shutil from pathlib import Path from modules.videos import captions @@ -46,11 +47,11 @@ class TestCaption(TestAPI): video1 = Video(title='scream', caption_path=str(self.vtt_path1)) session.add(video1) session.commit() - video1.caption = captions.get_video_captions(video1) + video1.caption = captions.get_captions(video1.caption_path.path) video2 = Video(title='bar', caption_path=str(self.vtt_path2)) session.add(video2) session.commit() - video2.caption = captions.get_video_captions(video2) + video2.caption = captions.get_captions(video2.caption_path.path) session.commit() session.flush() @@ -91,3 +92,13 @@ class TestCaption(TestAPI): # Matches video2.title select_textsearch('bar') self.assertEqual(curs.fetchall(), [[2, ]]) + + +def test_bad_caption(test_directory): + """A caption file that cannot be parsed is ignored.""" + bad_caption_file = PROJECT_DIR / 'test/bad_caption.en.vtt' + test_file = test_directory / 'bad_caption.en.vtt' + shutil.copy(bad_caption_file, test_file) + + result = captions.get_captions(test_file) + assert result is None diff --git a/requirements.txt b/requirements.txt index 9e681a9d..3127e2c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ sanic-testing==0.8.2 sanic==21.12.1 selenium==4.1.0 SQLAlchemy==1.3.22 -srt==3.5.0 +srt==3.5.1 websockets==10.1 webvtt-py==0.4.6 yt-dlp==2022.1.21 diff --git a/test/bad_caption.en.vtt b/test/bad_caption.en.vtt new file mode 100644 index 00000000..463b74ef --- /dev/null +++ b/test/bad_caption.en.vtt @@ -0,0 +1,8 @@ +WEBVTT +Kind: captions +Language: en + +00:00:00.070 --> 00:00:06.380 +this caption is bad + +00:05:08.650 --> 00:05:11.26 From 30ca95d627768e78e7dc8cb2496c64b40ea44c58 Mon Sep 17 00:00:00 2001 From: Roland Date: Mon, 31 Jan 2022 17:00:40 -0700 Subject: [PATCH 2/9] Creating files search api endpoint. --- modules/files/api.py | 24 ++++++++++++++++++++++-- modules/files/schema.py | 6 ++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/modules/files/api.py b/modules/files/api.py index daa296bc..49905bd3 100644 --- a/modules/files/api.py +++ b/modules/files/api.py @@ -6,8 +6,8 @@ from sanic import response from sanic_ext import validate from sanic_ext.extensions.openapi import openapi -from wrolpi.common import get_media_directory -from wrolpi.errors import InvalidFile +from wrolpi.common import get_media_directory, api_param_limiter +from wrolpi.errors import InvalidFile, ValidationError from wrolpi.root_api import get_blueprint, json_response from . import lib, schema @@ -61,3 +61,23 @@ async def delete_file(_: Request, body: schema.DeleteRequest): raise InvalidFile('file cannot be empty') lib.delete_file(body.file) return response.empty() + + +files_limit_limiter = api_param_limiter(100) +files_offset_limiter = api_param_limiter(100, 0) + + +@bp.post('/search') +@validate_doc( + 'Search files', + consumes=schema.FilesSearchRequest, + optional_body=True, +) +async def search(_: Request, data: dict): + try: + search_str = data.get('search_str') + domain = data.get('domain') + limit = files_limit_limiter(data.get('limit')) + offset = files_offset_limiter(data.get('offset')) + except Exception as e: + raise ValidationError('Unable to validate search queries') from e diff --git a/modules/files/schema.py b/modules/files/schema.py index 8952834c..76800b21 100644 --- a/modules/files/schema.py +++ b/modules/files/schema.py @@ -10,3 +10,9 @@ class FilesRequest: @dataclass class DeleteRequest: file: str + + +class FilesSearchRequest: + search_str = doc.String() + limit = doc.Integer() + offset = doc.Integer() From a5e8ef16b6a53b42731175b2899c06aeff0f9e1b Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 1 Feb 2022 12:22:29 -0700 Subject: [PATCH 3/9] Files are indexed by the words in their stem. Files can be searched via API. --- alembic/versions/36cae041ad31_files.py | 52 ++++++++++++++ modules/files/api.py | 26 ++++--- modules/files/lib.py | 96 +++++++++++++++++++++++++- modules/files/models.py | 34 +++++++++ modules/files/schema.py | 9 +-- modules/files/test/test_api.py | 40 +++++++++++ modules/files/test/test_lib.py | 15 ++++ requirements.txt | 1 + wrolpi/common.py | 8 +++ wrolpi/test/common.py | 30 ++++---- 10 files changed, 279 insertions(+), 32 deletions(-) create mode 100644 alembic/versions/36cae041ad31_files.py create mode 100644 modules/files/models.py diff --git a/alembic/versions/36cae041ad31_files.py b/alembic/versions/36cae041ad31_files.py new file mode 100644 index 00000000..dc8f8a2e --- /dev/null +++ b/alembic/versions/36cae041ad31_files.py @@ -0,0 +1,52 @@ +"""Files + +Revision ID: 36cae041ad31 +Revises: b97c1313f986 +Create Date: 2022-02-01 11:05:57.605541 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '36cae041ad31' +down_revision = 'b97c1313f986' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('''CREATE TABLE file ( + id BIGSERIAL PRIMARY KEY, + path TEXT NOT NULL, + title TEXT, + mimetype TEXT, + size INTEGER, + modification_datetime TIMESTAMPTZ, + textsearch tsvector GENERATED ALWAYS AS ( + setweight(to_tsvector('english'::regconfig, COALESCE(title, ''::text)), 'A'::"char") + ) stored + )''') + session.execute('CREATE INDEX file_path_idx ON file(path)') + session.execute('CREATE INDEX file_mimetype_idx ON file(textsearch)') + session.execute('CREATE INDEX file_textsearch_idx ON file(textsearch)') + if not DOCKERIZED: + session.execute('ALTER TABLE public.file OWNER TO wrolpi') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + + session.execute('DROP INDEX IF EXISTS file_textsearch_idx') + session.execute('DROP INDEX IF EXISTS file_mimetype_idx') + session.execute('DROP INDEX IF EXISTS file_path_idx') + + session.execute('DROP TABLE IF EXISTS file') diff --git a/modules/files/api.py b/modules/files/api.py index 49905bd3..45c78f92 100644 --- a/modules/files/api.py +++ b/modules/files/api.py @@ -7,7 +7,7 @@ from sanic_ext import validate from sanic_ext.extensions.openapi import openapi from wrolpi.common import get_media_directory, api_param_limiter -from wrolpi.errors import InvalidFile, ValidationError +from wrolpi.errors import InvalidFile from wrolpi.root_api import get_blueprint, json_response from . import lib, schema @@ -67,17 +67,15 @@ files_limit_limiter = api_param_limiter(100) files_offset_limiter = api_param_limiter(100, 0) +@bp.post('/refresh') +@openapi.description('Find and index all files') +async def refresh(_: Request): + lib.refresh_files() + return response.empty() + + @bp.post('/search') -@validate_doc( - 'Search files', - consumes=schema.FilesSearchRequest, - optional_body=True, -) -async def search(_: Request, data: dict): - try: - search_str = data.get('search_str') - domain = data.get('domain') - limit = files_limit_limiter(data.get('limit')) - offset = files_offset_limiter(data.get('offset')) - except Exception as e: - raise ValidationError('Unable to validate search queries') from e +@validate(schema.FilesSearchRequest) +async def search_files(_: Request, body: schema.FilesSearchRequest): + files, total = lib.search(body.search_str, body.limit, body.offset) + return json_response(dict(files=files, totals=dict(files=total))) diff --git a/modules/files/lib.py b/modules/files/lib.py index 6224be4e..b93f5fcf 100644 --- a/modules/files/lib.py +++ b/modules/files/lib.py @@ -1,8 +1,19 @@ +import asyncio +import re +from functools import wraps from pathlib import Path from typing import List -from wrolpi.common import get_media_directory, wrol_mode_check +import magic +import psycopg2 +from sqlalchemy.orm import Session + +from modules.files.models import File +from wrolpi.common import get_media_directory, wrol_mode_check, walk, chunks +from wrolpi.dates import from_timestamp +from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidFile +from wrolpi.vars import PYTEST def filter_parent_directories(directories: List[Path]) -> List[Path]: @@ -56,3 +67,86 @@ def delete_file(file: str): if file.is_dir() or not file.is_file(): raise InvalidFile(f'Invalid file {file}') file.unlink() + + +FILE_NAME_REGEX = re.compile(r'[_ ]') + + +def split_file_name(path: Path) -> List[str]: + """ + Split a file name into words. + + >>> split_file_name(Path('foo.mp4')) + ['foo'] + >>> split_file_name(Path('foo bar.mp4')) + ['foo', 'bar'] + """ + words = [str(i) for i in FILE_NAME_REGEX.split(path.stem)] + return words + + +def upsert_file(path: Path, session: Session) -> File: + file = session.query(File).filter_by(path=path).one_or_none() + if not file: + file = File(path=path) + session.add(file) + if not file.mimetype: + file.mimetype = magic.from_file(str(path), mime=True) + if not file.title: + file.title = split_file_name(path) + if not file.size or not file.modification_datetime: + stat = path.stat() + file.size = stat.st_size + file.modification_datetime = file.modification_datetime or from_timestamp(stat.st_mtime) + + return file + + +def _refresh_files(): + """Find and index all files""" + paths = filter(lambda i: i.is_file(), walk(get_media_directory())) + for chunk in chunks(paths, 20): + with get_db_session(commit=True) as session: + for path in chunk: + upsert_file(path, session) + + +@wraps(_refresh_files) +def refresh_files(): + """ + Schedule a refresh task if not testing. If testing, do a synchronous refresh. + """ + if PYTEST: + return _refresh_files() + + async def _(): + return _refresh_files() + + asyncio.create_task(_()) + + +def search(search_str: str, limit: int, offset: int): + with get_db_curs() as curs: + stmt = ''' + SELECT id, ts_rank_cd(textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total + FROM file + WHERE textsearch @@ websearch_to_tsquery(%(search_str)s) + OFFSET %(offset)s LIMIT %(limit)s + ''' + params = dict(search_str=search_str, offset=offset, limit=limit) + curs.execute(stmt, params) + + try: + results = list(map(dict, curs.fetchall())) + except psycopg2.ProgrammingError: + # No files. + return [], 0 + + total = results[0]['total'] if results else 0 + ranked_ids = [i['id'] for i in results] + + with get_db_session() as session: + results = get_ranked_models(ranked_ids, File, session) + results = [i.__json__() for i in results] + + return results, total diff --git a/modules/files/models.py b/modules/files/models.py new file mode 100644 index 00000000..979a5391 --- /dev/null +++ b/modules/files/models.py @@ -0,0 +1,34 @@ +from sqlalchemy import Integer, Column, String, Computed +from sqlalchemy.orm import deferred + +from wrolpi.common import Base, ModelHelper, get_media_directory, tsvector +from wrolpi.dates import TZDateTime +from wrolpi.media_path import MediaPathType + + +class File(ModelHelper, Base): + __tablename__ = 'file' + id = Column(Integer, primary_key=True) + + mimetype = Column(String) + modification_datetime = Column(TZDateTime) + path = Column(MediaPathType) + size = Column(Integer) + title = Column(String) + + textsearch = deferred( + Column(tsvector, Computed('''to_tsvector('english'::regconfig, (COALESCE(title, ''::text) || ' '::text))'''))) + + def __repr__(self): + return f'' + + def __json__(self): + d = dict( + id=self.id, + mimetype=self.mimetype, + modification_datetime=self.modification_datetime, + path=self.path.path.relative_to(get_media_directory()), + size=self.size, + title=self.title, + ) + return d diff --git a/modules/files/schema.py b/modules/files/schema.py index 76800b21..af2bf73b 100644 --- a/modules/files/schema.py +++ b/modules/files/schema.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import List +from typing import List, Optional @dataclass @@ -12,7 +12,8 @@ class DeleteRequest: file: str +@dataclass class FilesSearchRequest: - search_str = doc.String() - limit = doc.Integer() - offset = doc.Integer() + search_str: str + limit: Optional[int] = 20 + offset: Optional[int] = 0 diff --git a/modules/files/test/test_api.py b/modules/files/test/test_api.py index 0fb1ea41..1b09e281 100644 --- a/modules/files/test/test_api.py +++ b/modules/files/test/test_api.py @@ -6,6 +6,8 @@ from unittest import mock import pytest from wrolpi.errors import API_ERRORS, WROLModeEnabled +from wrolpi.test.common import assert_dict_contains +from wrolpi.vars import PROJECT_DIR def test_list_files_api(test_client, make_files_structure, test_directory): @@ -114,3 +116,41 @@ def test_delete_wrol_mode(test_client): request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'foo'})) assert response.status_code == HTTPStatus.FORBIDDEN assert response.json['code'] == API_ERRORS[WROLModeEnabled]['code'] + + +def do_search(test_client, search_str, total, expected): + content = json.dumps({'search_str': search_str}) + request, response = test_client.post('/api/files/search', content=content) + assert response.json['totals']['files'] == total + for file, expected in zip_longest(response.json['files'], expected): + assert_dict_contains(file, expected) + + +def test_files_search(test_session, test_client, make_files_structure): + # You can search an empty directory. + do_search(test_client, 'nothing', 0, []) + + files = [ + 'foo_is_the_name.txt', + 'archives/bar.txt', + 'baz.mp4', + 'baz two.mp4' + ] + foo, bar, baz, baz2 = make_files_structure(files) + foo.write_text('foo contents') + bar.write_text('the bar contents') + baz.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()) + baz2.write_bytes((PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()) + + # Refresh so files can be searched. + request, response = test_client.post('/api/files/refresh') + assert response.status_code == HTTPStatus.NO_CONTENT + + do_search(test_client, 'foo', 1, [dict(path='foo_is_the_name.txt', mimetype='text/plain', size=12)]) + do_search(test_client, 'bar', 1, [dict(path='archives/bar.txt', mimetype='text/plain', size=16)]) + do_search(test_client, 'baz', 2, [ + dict(path='baz two.mp4', mimetype='video/mp4', size=1055736), + dict(path='baz.mp4', mimetype='video/mp4', size=1055736), + ]) + do_search(test_client, 'two', 1, [dict(path='baz two.mp4', mimetype='video/mp4', size=1055736)]) + do_search(test_client, 'nothing', 0, []) diff --git a/modules/files/test/test_lib.py b/modules/files/test/test_lib.py index 1a53bb29..57088aed 100644 --- a/modules/files/test/test_lib.py +++ b/modules/files/test/test_lib.py @@ -122,3 +122,18 @@ def test_delete_file(make_files_structure, test_directory): lib.delete_file('does not exist') assert (test_directory / 'baz').is_dir() + + +@pytest.mark.parametrize( + 'path,expected', + [ + ('foo.mp4', ['foo']), + ('foo bar.txt', ['foo', 'bar']), + ('foo_bar.mp4', ['foo', 'bar']), + ('foo-bar', ['foo-bar']), + ('123 foo bar', ['123', 'foo', 'bar']), + ('123foo bar', ['123foo', 'bar']), + ] +) +def test_split_file_name(path, expected): + assert lib.split_file_name(Path(path)) == expected diff --git a/requirements.txt b/requirements.txt index 3127e2c4..e5954ff3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ Pint==0.18 psycopg2-binary==2.9.3 pytest-asyncio==0.17.2 pytest==6.2.5 +python-magic==0.4.24 pytz==2021.3 PyYAML==6.0 requests==2.27.1 diff --git a/wrolpi/common.py b/wrolpi/common.py index 19134a0a..9056e473 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -638,3 +638,11 @@ def zig_zag(low: ZIG_TYPE, high: ZIG_TYPE) -> Generator[ZIG_TYPE, None, None]: if num >= high: divisor *= 2 num = low + (diff / divisor) + + +def walk(path: Path) -> Generator[Path, None, None]: + """Walk a directory structure yielding all files and directories.""" + for path in path.iterdir(): + yield path + if path.is_dir(): + yield from walk(path) diff --git a/wrolpi/test/common.py b/wrolpi/test/common.py index a93bc3dd..efbdd4a5 100644 --- a/wrolpi/test/common.py +++ b/wrolpi/test/common.py @@ -76,19 +76,7 @@ class PytestCase: @classmethod def assertDictContains(cls, d1: dict, d2: dict): # noqa - if hasattr(d1, '__dict__'): - d1 = d1.__dict__ - if hasattr(d2, '__dict__'): - d2 = d2.__dict__ - - for k2 in d2.keys(): - assert d1, f'dict 1 is empty: {d1}' - assert d2, f'dict 1 is empty: {d2}' - assert k2 in d1, f'dict 1 does not contain {k2}' - if isinstance(d1[k2], dict): - cls.assertDictContains(d1[k2], d2[k2]) - else: - assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1' + assert_dict_contains(d1, d2) def assertError(self, response, http_status: int, code=None): # noqa self.assertEqual(response.status_code, http_status) @@ -217,3 +205,19 @@ async def get_all_ws_messages(ws) -> List[dict]: break messages.append(json.loads(message)) return messages + + +def assert_dict_contains(d1: dict, d2: dict): + if hasattr(d1, '__dict__'): + d1 = d1.__dict__ + if hasattr(d2, '__dict__'): + d2 = d2.__dict__ + + for k2 in d2.keys(): + assert d1, f'dict 1 is empty: {d1}' + assert d2, f'dict 1 is empty: {d2}' + assert k2 in d1, f'dict 1 does not contain {k2}' + if isinstance(d1[k2], dict): + assert_dict_contains(d1[k2], d2[k2]) + else: + assert d1[k2] == d2[k2], f'{k2} of value "{d1[k2]}" does not equal {d2[k2]} in dict 1' From 643c86e9a46ebe21eda4ff55b5f015f6aa29d4b0 Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 1 Feb 2022 14:11:14 -0700 Subject: [PATCH 4/9] Files can be searched. --- alembic/versions/36cae041ad31_files.py | 2 +- app/src/api.js | 13 +- app/src/components/Files.js | 192 +++++++++++++++---------- app/src/hooks/customHooks.js | 50 ++++++- modules/files/lib.py | 3 + modules/files/models.py | 11 +- modules/files/test/test_api.py | 3 + 7 files changed, 187 insertions(+), 87 deletions(-) diff --git a/alembic/versions/36cae041ad31_files.py b/alembic/versions/36cae041ad31_files.py index dc8f8a2e..df8a7dfd 100644 --- a/alembic/versions/36cae041ad31_files.py +++ b/alembic/versions/36cae041ad31_files.py @@ -31,7 +31,7 @@ def upgrade(): size INTEGER, modification_datetime TIMESTAMPTZ, textsearch tsvector GENERATED ALWAYS AS ( - setweight(to_tsvector('english'::regconfig, COALESCE(title, ''::text)), 'A'::"char") + setweight(to_tsvector('english'::regconfig, title), 'A'::"char") ) stored )''') session.execute('CREATE INDEX file_path_idx ON file(path)') diff --git a/app/src/api.js b/app/src/api.js index bcb80594..e410a8e2 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -352,19 +352,16 @@ export async function postDownload(urls, downloader) { return response; } - export async function killDownload(download_id) { let response = await apiPost(`${API_URI}/download/${download_id}/kill`); return response; } - export async function killDownloads() { let response = await apiPost(`${API_URI}/download/kill`); return response; } - export async function startDownloads() { let response = await apiPost(`${API_URI}/download/enable`); return response; @@ -375,6 +372,16 @@ export async function getDownloaders() { return await response.json(); } +export async function filesSearch(offset, limit, searchStr) { + const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)}; + const response = await apiPost(`${API_URI}/files/search`, body); + return await response.json(); +} + +export async function refreshFiles() { + return await apiPost(`${API_URI}/files/refresh`); +} + export async function getFiles(directories) { let body = {directories: directories || []}; let response = await apiPost(`${API_URI}/files`, body); diff --git a/app/src/components/Files.js b/app/src/components/Files.js index 149d8329..ceb8c02f 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -2,8 +2,10 @@ import React from "react"; import FileBrowser from 'react-keyed-file-browser'; import {Button, Confirm, Header, Icon, Modal} from "semantic-ui-react"; import 'react-keyed-file-browser/dist/react-keyed-file-browser.css'; -import {deleteFile, getFiles} from "../api"; -import {humanFileSize, PageContainer, secondsToDateTime} from "./Common"; +import {deleteFile, refreshFiles} from "../api"; +import Paginator, {humanFileSize, PageContainer, SearchInput, TabLinks} from "./Common"; +import {useBrowseFiles, useSearchFiles} from "../hooks/customHooks"; +import {Route} from "react-router-dom"; const icons = { File: , @@ -13,107 +15,143 @@ const icons = { PDF: , }; -class Files extends React.Component { - constructor(props) { - super(props); - this.state = { - deleteOpen: false, - files: [], - openFolders: [], - previewOpen: false, - selectedFile: null, - }; - } +export function Files(props) { + const {searchFiles, setPage, limit, totalPages, activePage, searchStr, setSearchStr} = useSearchFiles(50); + const {browseFiles, setOpenFolders, fetchFiles} = useBrowseFiles(); - async componentDidMount() { - await this.fetchFiles(); - } + const [deleteOpen, setDeleteOpen] = React.useState(false); + const [previewOpen, setPreviewOpen] = React.useState(false); + const [selectedFile, setSelectedFile] = React.useState(null); - fetchFiles = async () => { - let {files} = await getFiles(this.state.openFolders); - for (let i = 0; i < files.length; i++) { - files[i]['modified'] = secondsToDateTime(files[i]['modified']); - } - this.setState({files: files}); - } - - handleFolderChange = async (file, browserProps) => { + const handleFolderChange = async (file, browserProps) => { let openFolders = Object.keys(browserProps.openFolders); - this.setState({openFolders}, this.fetchFiles); + setOpenFolders(openFolders); } - onDownloadFile = async () => { + const onDownloadFile = async () => { // Open file in new tab/window. - window.open(`/media/${this.state.selectedFile.key}`); + window.open(`/media/${selectedFile.key}`); } - onSelectFile = async (file) => { - this.setState({selectedFile: file}, this.openPreview); + const onSelectFile = async (file) => { + setSelectedFile(file); + openPreview(); } - onDeleteFile = async () => { - await deleteFile(this.state.selectedFile.key); - await this.fetchFiles(); - this.closePreview(); + const onDeleteFile = async () => { + await deleteFile(selectedFile.key); + await fetchFiles(); + closePreview(); } - openPreview = () => { - this.setState({previewOpen: true}); + const openPreview = () => { + setPreviewOpen(true); } - closePreview = () => { - this.setState({previewOpen: false, selectedFile: null, deleteOpen: false}); + const closePreview = () => { + setPreviewOpen(false); + setSelectedFile(null); + setDeleteOpen(false); } - openDelete = () => { - this.setState({deleteOpen: true}); + const openDelete = () => { + setDeleteOpen(true); } - closeDelete = () => { - this.setState({deleteOpen: false}); + const closeDelete = () => { + setDeleteOpen(false); } - render() { - return <> - {this.state.selectedFile && - - File Preview - -
{this.state.selectedFile.name}
-

{humanFileSize(this.state.selectedFile.size)}

- - - - - - - } - <>} // Hide the preview that the 3rd party provided. - /> - + let selectedModal = <>; + if (selectedFile) { + selectedModal = ( + + File Preview + +
{selectedFile.name}
+

{humanFileSize(selectedFile.size)}

+ + + + + + + ); } + + let browser; + let pagination = null; + if (searchStr) { + browser = ( <>} // Hide the preview that the 3rd party provided. + />); + pagination = ( +
+ +
); + } else { + browser = ( <>} // Hide the preview that the 3rd party provided. + />); + } + + console.log('Files', limit); + return <> + + {selectedModal} + {browser} + {pagination} + +} + +function ManageFiles(props) { + return (<> + + + ) } export function FilesRoute(props) { + const links = [ + {text: 'Files', to: '/files', exact: true, key: 'files'}, + {text: 'Manage', to: '/files/manage', exact: true, key: 'manage'}, + ]; + return ( - + + + ); } diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 95def379..b19341b1 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -1,5 +1,5 @@ import {useEffect, useState} from "react"; -import {fetchDomains, getVersion, searchArchives, searchVideos} from "../api"; +import {fetchDomains, filesSearch, getFiles, getVersion, searchArchives, searchVideos} from "../api"; import {useHistory} from "react-router-dom"; export const useSearchParam = (key, defaultValue = null) => { @@ -130,3 +130,51 @@ export const useVersion = () => { return version; } + +export const useSearchFiles = ({defaultLimit = 50}) => { + const [searchFiles, setSearchFiles] = useState([]); + const [totalPages, setTotalPages] = useState(0); + const [offset, setOffset] = useSearchParam('o'); + const [limit, setLimit] = useSearchParam('l', defaultLimit); + const [searchStr, setSearchStr] = useSearchParam('q'); + const [activePage, setActivePage] = useState(1); + + const setPage = (i) => { + i = parseInt(i); + let l = parseInt(limit); + setOffset((l * i) - l); + setActivePage(i); + } + + const localSearchFiles = async () => { + setSearchFiles([]); + setTotalPages(0); + if (searchStr) { + let {files, totals} = await filesSearch(offset, limit, searchStr); + setSearchFiles(files); + setTotalPages(Math.floor(totals['files'] / limit) + 1); + } + } + + useEffect(() => { + localSearchFiles(); + }, [searchStr, limit, offset, activePage]); + + return {searchFiles, totalPages, limit, setLimit, setOffset, searchStr, setSearchStr, activePage, setPage}; +} + +export const useBrowseFiles = () => { + const [browseFiles, setBrowseFiles] = useState([]); + const [openFolders, setOpenFolders] = useState([]); + + const fetchFiles = async () => { + let {files} = await getFiles(openFolders); + setBrowseFiles(files); + } + + useEffect(() => { + fetchFiles(); + }, [openFolders]) + + return {browseFiles, openFolders, setOpenFolders, fetchFiles}; +} diff --git a/modules/files/lib.py b/modules/files/lib.py index b93f5fcf..ba318ebc 100644 --- a/modules/files/lib.py +++ b/modules/files/lib.py @@ -126,6 +126,9 @@ def refresh_files(): def search(search_str: str, limit: int, offset: int): + """ + Search the "title" of each file. The title is the words in the file's stem. + """ with get_db_curs() as curs: stmt = ''' SELECT id, ts_rank_cd(textsearch, websearch_to_tsquery(%(search_str)s)), COUNT(*) OVER() AS total diff --git a/modules/files/models.py b/modules/files/models.py index 979a5391..e116fc7b 100644 --- a/modules/files/models.py +++ b/modules/files/models.py @@ -9,26 +9,27 @@ from wrolpi.media_path import MediaPathType class File(ModelHelper, Base): __tablename__ = 'file' id = Column(Integer, primary_key=True) + path = Column(MediaPathType) mimetype = Column(String) modification_datetime = Column(TZDateTime) - path = Column(MediaPathType) size = Column(Integer) - title = Column(String) + title = deferred(Column(String)) # used by textsearch column textsearch = deferred( - Column(tsvector, Computed('''to_tsvector('english'::regconfig, (COALESCE(title, ''::text) || ' '::text))'''))) + Column(tsvector, Computed('''to_tsvector('english'::regconfig, title)'''))) def __repr__(self): return f'' def __json__(self): + path = self.path.path.relative_to(get_media_directory()) d = dict( id=self.id, mimetype=self.mimetype, modification_datetime=self.modification_datetime, - path=self.path.path.relative_to(get_media_directory()), + path=path, size=self.size, - title=self.title, + key=path, ) return d diff --git a/modules/files/test/test_api.py b/modules/files/test/test_api.py index 1b09e281..3931282b 100644 --- a/modules/files/test/test_api.py +++ b/modules/files/test/test_api.py @@ -124,12 +124,15 @@ def do_search(test_client, search_str, total, expected): assert response.json['totals']['files'] == total for file, expected in zip_longest(response.json['files'], expected): assert_dict_contains(file, expected) + # FileBrowser in React requires a key. We use the path. + assert file['path'] == file['key'] def test_files_search(test_session, test_client, make_files_structure): # You can search an empty directory. do_search(test_client, 'nothing', 0, []) + # Create files in the temporary directory. Add some contents so the mimetype can be tested. files = [ 'foo_is_the_name.txt', 'archives/bar.txt', From 79f2359e83d5db20de191c4455dd65b4ebfca0f6 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 2 Feb 2022 08:24:34 -0700 Subject: [PATCH 5/9] Redoing Paginator with custom hook. Keeping Old Paginator for Videos until it is updated with hooks. --- app/src/components/Archive.js | 10 +++------ app/src/components/Common.js | 27 +++++++++++++++++++++- app/src/components/Files.js | 5 ++--- app/src/components/Videos.js | 4 ++-- app/src/hooks/customHooks.js | 42 +++++++++++++++++------------------ 5 files changed, 54 insertions(+), 34 deletions(-) diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index 78add301..6e0bd981 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -1,10 +1,10 @@ import React, {useState} from "react"; import {Card, Confirm, Container, Icon, Image, Placeholder} from "semantic-ui-react"; -import Paginator, { +import { CardGroupCentered, ClearButton, ExternalLink, - PageContainer, + PageContainer, Paginator, SearchInput, TabLinks, textEllipsis, @@ -237,11 +237,7 @@ export function Archives() { setDomain={setDomain} />
- +
) diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 5ee771a6..6683d7ee 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -5,6 +5,7 @@ import Button from "semantic-ui-react/dist/commonjs/elements/Button"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; import {getConfig} from "../api"; +import {useSearchParams} from "../hooks/customHooks"; export const API_URI = `http://${window.location.host}/api`; export const VIDEOS_API = `${API_URI}/videos`; @@ -12,7 +13,7 @@ export const ARCHIVES_API = `${API_URI}/archive`; export const OTP_API = `${API_URI}/otp`; export const DEFAULT_LIMIT = 20; -export default class Paginator extends React.Component { +export default class OldPaginator extends React.Component { state = { boundaryRange: 1, siblingRange: 2, @@ -54,6 +55,30 @@ export default class Paginator extends React.Component { } } +export function Paginator(props) { + let {activePage, setPage} = useSearchParams(props.defaultLimit || 20); + + const handlePaginationChange = (e, {activePage}) => { + setPage(activePage); + } + + return ( + + ) +} + export function secondsToDuration(video) { let duration = video.duration; let hours = Math.floor(duration / 3600); diff --git a/app/src/components/Files.js b/app/src/components/Files.js index ceb8c02f..5fb86270 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -3,7 +3,7 @@ import FileBrowser from 'react-keyed-file-browser'; import {Button, Confirm, Header, Icon, Modal} from "semantic-ui-react"; import 'react-keyed-file-browser/dist/react-keyed-file-browser.css'; import {deleteFile, refreshFiles} from "../api"; -import Paginator, {humanFileSize, PageContainer, SearchInput, TabLinks} from "./Common"; +import {Paginator, humanFileSize, PageContainer, SearchInput, TabLinks} from "./Common"; import {useBrowseFiles, useSearchFiles} from "../hooks/customHooks"; import {Route} from "react-router-dom"; @@ -16,7 +16,7 @@ const icons = { }; export function Files(props) { - const {searchFiles, setPage, limit, totalPages, activePage, searchStr, setSearchStr} = useSearchFiles(50); + const {searchFiles, setPage, totalPages, activePage, searchStr, setSearchStr} = useSearchFiles(50); const {browseFiles, setOpenFolders, fetchFiles} = useBrowseFiles(); const [deleteOpen, setDeleteOpen] = React.useState(false); @@ -119,7 +119,6 @@ export function Files(props) { />); } - console.log('Files', limit); return <> {selectedModal} diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index 06bf309b..2d21da7d 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -1,6 +1,6 @@ import React from 'react'; import {Link, Route} from "react-router-dom"; -import Paginator, { +import OldPaginator, { changePageHistory, DEFAULT_LIMIT, defaultSearchOrder, @@ -401,7 +401,7 @@ class Videos extends React.Component { if (totalPages) { pagination = (
- { return [value, setValue]; } +export const useSearchParams = ({defaultLimit = 20}) => { + let [limit, setLimit] = useSearchParam('l', defaultLimit); + let [offset, setOffset] = useSearchParam('o'); + let [searchStr, setSearchStr] = useSearchParam('q'); + + let [activePage, setActivePage] = useState(1); + + const setPage = (i) => { + i = parseInt(i); + let l = parseInt(limit); + setOffset((l * i) - l); + setActivePage(i); + } + + return {limit, setLimit, offset, setOffset, searchStr, setSearchStr, activePage, setPage}; +} + export const useSearch = () => { let [searchStr, setSearchStr] = useSearchParam('q'); @@ -73,17 +90,8 @@ export const useArchives = (defaultLimit = 20) => { const [totalPages, setTotalPages] = useState(0); const [activePage, setActivePage] = useState(1); - let [offset, setOffset] = useSearchParam('o'); - let [limit, setLimit] = useSearchParam('l', defaultLimit); + let {limit, setLimit, offset, setOffset, searchStr, setSearchStr, setPage} = useSearchParams(defaultLimit); let [domain, setDomain] = useSearchParam('domain'); - let [searchStr, setSearchStr] = useSearchParam('q'); - - const setPage = (i) => { - i = parseInt(i); - let l = parseInt(limit); - setOffset((l * i) - l); - setActivePage(i); - } const search = async (term) => { setArchives({archives: null, total: 0}); @@ -132,19 +140,11 @@ export const useVersion = () => { } export const useSearchFiles = ({defaultLimit = 50}) => { + let {limit, setLimit, offset, setOffset, searchStr, setSearchStr, activePage, setPage} = + useSearchParams(defaultLimit); + const [searchFiles, setSearchFiles] = useState([]); const [totalPages, setTotalPages] = useState(0); - const [offset, setOffset] = useSearchParam('o'); - const [limit, setLimit] = useSearchParam('l', defaultLimit); - const [searchStr, setSearchStr] = useSearchParam('q'); - const [activePage, setActivePage] = useState(1); - - const setPage = (i) => { - i = parseInt(i); - let l = parseInt(limit); - setOffset((l * i) - l); - setActivePage(i); - } const localSearchFiles = async () => { setSearchFiles([]); From 22f603e567c270cc821b6490bbadd899af7b1617 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 2 Feb 2022 08:31:13 -0700 Subject: [PATCH 6/9] Adding log when refreshing files. --- modules/files/lib.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/files/lib.py b/modules/files/lib.py index ba318ebc..24cab77e 100644 --- a/modules/files/lib.py +++ b/modules/files/lib.py @@ -9,12 +9,14 @@ import psycopg2 from sqlalchemy.orm import Session from modules.files.models import File -from wrolpi.common import get_media_directory, wrol_mode_check, walk, chunks +from wrolpi.common import get_media_directory, wrol_mode_check, walk, chunks, logger from wrolpi.dates import from_timestamp from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidFile from wrolpi.vars import PYTEST +logger = logger.getChild(__name__) + def filter_parent_directories(directories: List[Path]) -> List[Path]: """ @@ -86,6 +88,7 @@ def split_file_name(path: Path) -> List[str]: def upsert_file(path: Path, session: Session) -> File: + """Update/insert a File in the DB. Gather metadata about it.""" file = session.query(File).filter_by(path=path).one_or_none() if not file: file = File(path=path) @@ -104,6 +107,7 @@ def upsert_file(path: Path, session: Session) -> File: def _refresh_files(): """Find and index all files""" + logger.info('Refreshing Files') paths = filter(lambda i: i.is_file(), walk(get_media_directory())) for chunk in chunks(paths, 20): with get_db_session(commit=True) as session: @@ -113,9 +117,7 @@ def _refresh_files(): @wraps(_refresh_files) def refresh_files(): - """ - Schedule a refresh task if not testing. If testing, do a synchronous refresh. - """ + """Schedule a refresh task if not testing. If testing, do a synchronous refresh.""" if PYTEST: return _refresh_files() From 812369fdb325704e2abc7f80b85d9823e13dcb4a Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 2 Feb 2022 08:50:03 -0700 Subject: [PATCH 7/9] Refresh files removed files that were deleted. --- alembic/versions/36cae041ad31_files.py | 3 ++- modules/files/lib.py | 15 +++++++++++- modules/files/models.py | 1 + modules/files/test/test_lib.py | 32 ++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/alembic/versions/36cae041ad31_files.py b/alembic/versions/36cae041ad31_files.py index df8a7dfd..8e69db38 100644 --- a/alembic/versions/36cae041ad31_files.py +++ b/alembic/versions/36cae041ad31_files.py @@ -32,7 +32,8 @@ def upgrade(): modification_datetime TIMESTAMPTZ, textsearch tsvector GENERATED ALWAYS AS ( setweight(to_tsvector('english'::regconfig, title), 'A'::"char") - ) stored + ) stored, + idempotency TEXT )''') session.execute('CREATE INDEX file_path_idx ON file(path)') session.execute('CREATE INDEX file_mimetype_idx ON file(textsearch)') diff --git a/modules/files/lib.py b/modules/files/lib.py index 24cab77e..45a15108 100644 --- a/modules/files/lib.py +++ b/modules/files/lib.py @@ -3,6 +3,7 @@ import re from functools import wraps from pathlib import Path from typing import List +from uuid import uuid4 import magic import psycopg2 @@ -17,6 +18,8 @@ from wrolpi.vars import PYTEST logger = logger.getChild(__name__) +__all__ = ['list_files', 'delete_file', 'split_file_name', 'upsert_file', 'refresh_files', 'search'] + def filter_parent_directories(directories: List[Path]) -> List[Path]: """ @@ -108,11 +111,21 @@ def upsert_file(path: Path, session: Session) -> File: def _refresh_files(): """Find and index all files""" logger.info('Refreshing Files') + # Mark all files as unverified. Any records that are unverified after refresh will be deleted. + with get_db_curs(commit=True) as curs: + curs.execute('UPDATE file SET idempotency=null') + paths = filter(lambda i: i.is_file(), walk(get_media_directory())) + idempotency = str(uuid4()) for chunk in chunks(paths, 20): with get_db_session(commit=True) as session: for path in chunk: - upsert_file(path, session) + file = upsert_file(path, session) + file.idempotency = idempotency + + # Remove any records where the file no longer exists. + with get_db_curs(commit=True) as curs: + curs.execute('DELETE FROM file WHERE idempotency IS NULL') @wraps(_refresh_files) diff --git a/modules/files/models.py b/modules/files/models.py index e116fc7b..7b8207b4 100644 --- a/modules/files/models.py +++ b/modules/files/models.py @@ -11,6 +11,7 @@ class File(ModelHelper, Base): id = Column(Integer, primary_key=True) path = Column(MediaPathType) + idempotency = Column(String) mimetype = Column(String) modification_datetime = Column(TZDateTime) size = Column(Integer) diff --git a/modules/files/test/test_lib.py b/modules/files/test/test_lib.py index 57088aed..a15862d5 100644 --- a/modules/files/test/test_lib.py +++ b/modules/files/test/test_lib.py @@ -1,9 +1,12 @@ from pathlib import Path +from typing import List, Iterable import pytest from modules.files import lib +from modules.files.models import File from wrolpi.errors import InvalidFile +from wrolpi.media_path import MediaPathType def test_list_files(make_files_structure, test_directory): @@ -137,3 +140,32 @@ def test_delete_file(make_files_structure, test_directory): ) def test_split_file_name(path, expected): assert lib.split_file_name(Path(path)) == expected + + +def test_refresh_files(test_session, make_files_structure, test_directory): + files = make_files_structure([ + 'foo.txt', + 'bar.txt', + 'baz.txt', + ]) + foo, bar, baz = files + + def get_relative_strs(files_: Iterable[MediaPathType]) -> List[str]: + return sorted([str(i.path.path.relative_to(test_directory)) for i in files_]) + + lib.refresh_files() + results = test_session.query(File) + assert get_relative_strs(results) == ['bar.txt', 'baz.txt', 'foo.txt'] + + baz.unlink() + + lib.refresh_files() + results = test_session.query(File) + assert get_relative_strs(results) == ['bar.txt', 'foo.txt'] + + foo.unlink() + bar.unlink() + + lib.refresh_files() + results = test_session.query(File) + assert get_relative_strs(results) == [] From 2053dc46f97ef77afdaa36694b2f9d7a0aeaf440 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 2 Feb 2022 09:12:01 -0700 Subject: [PATCH 8/9] Replace modified timestamps with datetimes. --- app/src/hooks/customHooks.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 501a0087..4fe13be5 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -1,6 +1,7 @@ import {useEffect, useState} from "react"; import {fetchDomains, filesSearch, getFiles, getVersion, searchArchives, searchVideos} from "../api"; import {useHistory} from "react-router-dom"; +import {secondsToDateTime} from "../components/Common"; export const useSearchParam = (key, defaultValue = null) => { // Get a window.location.search param. @@ -139,6 +140,15 @@ export const useVersion = () => { return version; } +const replaceFileDatetimes = (files) => { + for (let i = 0; i < files.length; i++) { + if (files[i]['modified']) { + files[i]['modified'] = secondsToDateTime(files[i]['modified']); + } + } + return files; +} + export const useSearchFiles = ({defaultLimit = 50}) => { let {limit, setLimit, offset, setOffset, searchStr, setSearchStr, activePage, setPage} = useSearchParams(defaultLimit); @@ -151,6 +161,7 @@ export const useSearchFiles = ({defaultLimit = 50}) => { setTotalPages(0); if (searchStr) { let {files, totals} = await filesSearch(offset, limit, searchStr); + files = replaceFileDatetimes(files); setSearchFiles(files); setTotalPages(Math.floor(totals['files'] / limit) + 1); } @@ -169,6 +180,7 @@ export const useBrowseFiles = () => { const fetchFiles = async () => { let {files} = await getFiles(openFolders); + files = replaceFileDatetimes(files); setBrowseFiles(files); } From 3f08bce9eaafdbe3d57930e6f5dccefc7b559591 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 2 Feb 2022 14:02:51 -0700 Subject: [PATCH 9/9] Removing consolidated hook because values were not being passed up. --- app/src/api.js | 35 ++++++++++++-- app/src/components/Admin.js | 6 +-- app/src/components/Archive.js | 16 +++---- app/src/components/Common.js | 21 ++++----- app/src/components/Files.js | 4 +- app/src/components/Inventory.js | 4 +- app/src/components/Nav.js | 24 +++++----- app/src/components/Videos.js | 8 ++-- app/src/hooks/customHooks.js | 82 ++++++++++++++------------------- modules/files/lib.py | 2 + modules/files/models.py | 2 +- 11 files changed, 106 insertions(+), 98 deletions(-) diff --git a/app/src/api.js b/app/src/api.js index e410a8e2..0cf521b2 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -1,4 +1,4 @@ -import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, OTP_API, VIDEOS_API} from "./components/Common"; +import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, OTP_API, secondsToDateTime, VIDEOS_API} from "./components/Common"; import {toast} from 'react-semantic-toasts'; function timeoutPromise(ms, promise) { @@ -303,8 +303,8 @@ export async function deleteArchive(archive_id) { export async function searchArchives(offset, limit, domain = null, searchStr = null) { // Build a search query to retrieve a list of videos from the API - offset = offset || 0; - limit = limit || DEFAULT_LIMIT; + offset = parseInt(offset || 0); + limit = parseInt(limit || DEFAULT_LIMIT); let body = {offset, limit}; if (domain) { body['domain'] = domain; @@ -372,10 +372,33 @@ export async function getDownloaders() { return await response.json(); } +const replaceFileDatetimes = (files) => { + for (let i = 0; i < files.length; i++) { + let file = files[i]['modified']; + if (file['modified']) { + files[i]['modified'] = secondsToDateTime(file['modified']); + } + } + return files; +} + export async function filesSearch(offset, limit, searchStr) { const body = {search_str: searchStr, offset: parseInt(offset), limit: parseInt(limit)}; const response = await apiPost(`${API_URI}/files/search`, body); - return await response.json(); + + if (response.status === 200) { + let data = await response.json(); + let [files, total] = [data['files'], data['totals']['files']]; + files = replaceFileDatetimes(files); + return [files, total]; + } else { + toast({ + type: 'error', + title: 'Unable to search files', + description: 'Cannot search files. See server logs.', + time: 5000, + }); + } } export async function refreshFiles() { @@ -385,7 +408,9 @@ export async function refreshFiles() { export async function getFiles(directories) { let body = {directories: directories || []}; let response = await apiPost(`${API_URI}/files`, body); - return await response.json(); + let {files} = await response.json(); + files = replaceFileDatetimes(files); + return files; } export async function deleteFile(file) { diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js index 22687488..811c10f2 100644 --- a/app/src/components/Admin.js +++ b/app/src/components/Admin.js @@ -443,9 +443,9 @@ class Downloads extends React.Component { export default function Admin(props) { const links = [ - {text: 'Downloads', to: '/admin', exact: true}, - {text: 'Settings', to: '/admin/settings'}, - {text: 'WROL Mode', to: '/admin/wrol'}, + {text: 'Downloads', to: '/admin', exact: true, key: 'admin'}, + {text: 'Settings', to: '/admin/settings', key: 'settings'}, + {text: 'WROL Mode', to: '/admin/wrol', key: 'wrol'}, ]; return ( diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js index 6e0bd981..c36de3fb 100644 --- a/app/src/components/Archive.js +++ b/app/src/components/Archive.js @@ -4,7 +4,8 @@ import { CardGroupCentered, ClearButton, ExternalLink, - PageContainer, Paginator, + PageContainer, + Paginator, SearchInput, TabLinks, textEllipsis, @@ -202,9 +203,8 @@ export function ArchivesList({archives, searchStr, syncArchive, localDeleteArchi } export function Archives() { - const {archivesData, setPage, totalPages, searchStr, activePage, setSearchStr, domain, setDomain, search} = - useArchives(); - const {archives} = archivesData; + const {archives, setPage, totalPages, searchStr, activePage, setSearchStr, domain, setDomain, search} = + useArchives(20); const syncArchive = async (url) => { await postDownload(url, 'archive'); @@ -237,7 +237,7 @@ export function Archives() { setDomain={setDomain} />
- +
) @@ -319,9 +319,9 @@ function Domains() { export function ArchiveRoute(props) { const links = [ - {text: 'Archives', to: '/archive', exact: true}, - {text: 'Domains', to: '/archive/domains'}, - {text: 'Manage', to: '/archive/manage'}, + {text: 'Archives', to: '/archive', exact: true, key: 'archive'}, + {text: 'Domains', to: '/archive/domains', key: 'domains'}, + {text: 'Manage', to: '/archive/manage', key: 'manage'}, ]; return diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 6683d7ee..cd6f2e8a 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -5,7 +5,6 @@ import Button from "semantic-ui-react/dist/commonjs/elements/Button"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; import {getConfig} from "../api"; -import {useSearchParams} from "../hooks/customHooks"; export const API_URI = `http://${window.location.host}/api`; export const VIDEOS_API = `${API_URI}/videos`; @@ -55,26 +54,21 @@ export default class OldPaginator extends React.Component { } } -export function Paginator(props) { - let {activePage, setPage} = useSearchParams(props.defaultLimit || 20); - - const handlePaginationChange = (e, {activePage}) => { - setPage(activePage); +export function Paginator({activePage, onPageChange, totalPages, showFirstAndLast}) { + const handlePageChange = (e, {activePage}) => { + onPageChange(activePage); } return ( ) } @@ -607,6 +601,7 @@ export function TabLinks({links}) { className={'item'} exact={link.exact || false} style={{padding: '1em'}} + key={link.key} > {link.text} diff --git a/app/src/components/Files.js b/app/src/components/Files.js index 5fb86270..e40f5949 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -15,7 +15,7 @@ const icons = { PDF: , }; -export function Files(props) { +export function Files() { const {searchFiles, setPage, totalPages, activePage, searchStr, setSearchStr} = useSearchFiles(50); const {browseFiles, setOpenFolders, fetchFiles} = useBrowseFiles(); @@ -127,7 +127,7 @@ export function Files(props) { } -function ManageFiles(props) { +function ManageFiles() { return (<>