From f3439c2bd5756e625aec455d97d46b1a2fd3d47c Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 21 Jul 2024 15:04:56 -0600 Subject: [PATCH] Adding feature so that tags can be applied when uploading files. --- app/src/components/Upload.js | 4 +++- app/src/hooks/customHooks.js | 29 +++++++++++++++++++++++++++-- modules/videos/lib.py | 3 +-- wrolpi/conftest.py | 3 +++ wrolpi/files/api.py | 13 ++++++++++++- wrolpi/files/lib.py | 27 +++++++++++++++++++++++++-- wrolpi/files/test/test_api.py | 13 +++++++++---- 7 files changed, 80 insertions(+), 12 deletions(-) diff --git a/app/src/components/Upload.js b/app/src/components/Upload.js index 71262162..9423c559 100644 --- a/app/src/components/Upload.js +++ b/app/src/components/Upload.js @@ -9,9 +9,10 @@ import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import {Form as SForm} from "semantic-ui-react"; export function Upload({disabled}) { - const {setFiles, progresses, destination, setDestination, doClear} = useUploadFile(); const {t} = React.useContext(ThemeContext); + const {setFiles, progresses, destination, setDestination, doClear, tagsSelector} = useUploadFile(); + const onDrop = React.useCallback(async (acceptedFiles) => { if (acceptedFiles && acceptedFiles.length > 0) { setFiles(acceptedFiles); @@ -70,6 +71,7 @@ export function Upload({disabled}) { + {tagsSelector} diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 7e47eddb..c306695c 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -30,6 +30,7 @@ import {QueryContext, SettingsContext, StatusContext} from "../contexts/contexts import {toast} from "react-semantic-toasts-2"; import {useSearch} from "../components/Search"; import _ from "lodash"; +import {TagsSelector} from "../Tags"; const calculatePage = (offset, limit) => { return offset && limit ? Math.round((offset / limit) + 1) : 1; @@ -997,6 +998,7 @@ export const useUploadFile = () => { const [files, setFiles] = useState([]); const [progresses, setProgresses] = useState({}); const [destination, setDestination] = useState(''); + const [tagNames, setTagNames] = React.useState([]); const handleProgress = (name, chunk, totalChunks, status, type) => { const percent = Math.round((100 * chunk) / totalChunks); @@ -1031,8 +1033,11 @@ export const useUploadFile = () => { // Send the size that we're actually sending. formData.append('chunkSize', chunk.size.toString()); formData.append('chunk', chunk); + for (let i = 0; i < tagNames.length; i++) { + formData.append('tagNames', tagNames[i]); + } - console.debug(`file upload: tries=${tries} chunkNum=${chunkNum} totalChunks=${totalChunks} chunkSize=${chunk.size} destination=${destination}`); + console.debug(`file upload: tries=${tries} chunkNum=${chunkNum} totalChunks=${totalChunks} chunkSize=${chunk.size} destination=${destination} tagNames=${tagNames}`); const xhr = new XMLHttpRequest(); xhr.open('POST', '/api/files/upload', true); @@ -1101,7 +1106,27 @@ export const useUploadFile = () => { doUpload() }, [JSON.stringify(files)]); - return {files, setFiles: handleFilesChange, progresses, destination, setDestination, doClear, doUpload} + const handleAddTag = (tagName) => { + setTagNames([...tagNames, tagName]); + } + + const handleUntag = (tagName) => { + setTagNames(tagNames.filter(i => i !== tagName)); + } + + const tagsSelector = ; + + return { + destination, + doClear, + doUpload, + files, + progresses, + setDestination, + setFiles: handleFilesChange, + tagNames, + tagsSelector, + } } export const useSearchZim = (searchStr, zimId, active, activeTags, defaultLimit = 10) => { diff --git a/modules/videos/lib.py b/modules/videos/lib.py index ffb3c8fe..3a272d57 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -489,8 +489,7 @@ def link_channel_and_downloads(session: Session): # Get any Downloads for a Channel's RSS feed. rss_url = channel.get_rss_url() - download = downloads_by_url.get(rss_url) if rss_url else None - if download: + if rss_url and (download := downloads_by_url.get(rss_url)): download.channel_id = channel.id # Associate any Download which shares a Channel's URL. diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index aa6c5010..0fb31038 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -719,6 +719,9 @@ def make_multipart_form(): name, value, filename = form['name'], form['value'], form.get('filename') if filename: new_forms.append(f'Content-Disposition: form-data; name="{name}"; filename="{filename}"\n\n{value}\n') + elif isinstance(value, list): + for part in value: + new_forms.append(f'Content-Disposition: form-data; name="{name}"\n\n{part}\n') else: new_forms.append(f'Content-Disposition: form-data; name="{name}"\n\n{value}\n') body = f'\n{boundary}'.join(new_forms) diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index b129554e..841f81e2 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -13,6 +13,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileC from . import lib, schema from ..api_utils import json_response, api_app from ..schema import JSONErrorResponse +from ..tags import Tag from ..vars import PYTEST files_bp = Blueprint('Files', '/api/files') @@ -319,6 +320,15 @@ async def post_upload(request: Request): except Exception: raise FileUploadFailed('totalChunks integer is required') + tag_names = request.form.getlist('tagNames') + if tag_names: + if not isinstance(tag_names, list): + raise FileUploadFailed('tag_names must be a list') + tag_names = [str(i) for i in tag_names] + for tag_name in tag_names: + if not Tag.find_by_name(tag_name): + raise FileUploadFailed(f'Tag does not exist: {tag_name}') + filename = pathlib.Path(filename.lstrip('/')) output = destination / filename output_str = str(output) @@ -372,7 +382,8 @@ async def post_upload(request: Request): if chunk_num == total_chunks: # File upload is complete. del api_app.shared_ctx.uploaded_files[output_str] - background_task(lib.refresh_files([output.parent])) + # Upsert this new file (and any related files) into the DB. + background_task(lib.upsert_file(output, tag_names=tag_names)) return response.empty(HTTPStatus.CREATED) # Request the next chunk. diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index c6ffc5cd..5d56bc56 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -11,7 +11,7 @@ import shutil import subprocess from itertools import zip_longest from pathlib import Path -from typing import List, Tuple, Union, Dict, Generator +from typing import List, Tuple, Union, Dict, Generator, Iterable import cachetools.func import psycopg2 @@ -319,7 +319,7 @@ def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]: return files -def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> pathlib.Path: +def get_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]]) -> pathlib.Path: """Given a list of files, return the file that we can model or index. @raise NoPrimaryFile: If not primary file can be found.""" @@ -450,6 +450,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime): def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]: + """Return a new list which does not contain any file paths that are in ignored directories.""" ignored_directories = list(map(str, get_wrolpi_config().ignored_directories)) files = [i for i in files if not any(str(i).startswith(j) for j in ignored_directories)] return files @@ -1370,3 +1371,25 @@ def remove_ignored_directory(directory: Union[pathlib.Path, str]): get_wrolpi_config().ignored_directories = ignored_directories else: raise UnknownDirectory('Directory is not ignored') + + +@optional_session(commit=True) +async def upsert_file(file: pathlib.Path | str, session: Session = None, tag_names: List[str] = None): + """Insert/update all files in the provided file's FileGroup.""" + # Update/Insert all files in the FileGroup. + paths = glob_shared_stem(pathlib.Path(file)) + # Remove any ignored files. + paths = remove_files_in_ignored_directories(paths) + if paths: + file_group = FileGroup.from_paths(session, *paths) + # Re-index the contents of the file. + session.add(file_group) + session.flush([file_group, ]) + file_group.do_index() + + for tag_name in (tag_names or []): + if tag_name not in file_group.tag_names: + tag = Tag.find_by_name(tag_name) + file_group.add_tag(tag) + else: + logger.warning('upsert_file called, but all files are in ignored directories!') diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index 6a134cf9..a77c26e6 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -425,9 +425,10 @@ def test_post_upload_directory(test_session, test_client, test_directory, make_f assert test_session.query(FileGroup).count() == 1 -def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form): +def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form, tag_factory): """A file can be uploaded in chunks directly to the destination.""" make_files_structure(['uploads/']) + tag1, tag2 = tag_factory(), tag_factory() forms = [ dict(name='chunkNumber', value='0'), @@ -435,7 +436,8 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc dict(name='totalChunks', value='1'), dict(name='destination', value='uploads'), dict(name='chunkSize', value='3'), - dict(name='chunk', value='foo', filename='chunk') + dict(name='chunk', value='foo', filename='chunk'), + dict(name='tagNames', value=[tag1.name, tag2.name]), ] body = make_multipart_form(forms) request, response = test_client.post('/api/files/upload', content=body, @@ -453,7 +455,8 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc dict(name='totalChunks', value='1'), dict(name='destination', value='uploads'), dict(name='chunkSize', value='3'), - dict(name='chunk', value='bar', filename='chunk') + dict(name='chunk', value='bar', filename='chunk'), + dict(name='tagNames', value=[tag1.name, tag2.name]), ] body = make_multipart_form(forms) request, response = test_client.post('/api/files/upload', content=body, @@ -465,7 +468,9 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc assert (test_directory / 'uploads/foo.txt').is_file() assert (test_directory / 'uploads/foo.txt').read_text() == 'foobar' - assert test_session.query(FileGroup).count() == 1 + file_group: FileGroup = test_session.query(FileGroup).one() + assert set(file_group.tag_names) == {tag1.name, tag2.name}, 'Two tags should be applied.' + assert file_group.indexed, 'File should be indexed after upload.' @pytest.mark.asyncio