diff --git a/app/src/components/Upload.js b/app/src/components/Upload.js
index 71262162..9423c559 100644
--- a/app/src/components/Upload.js
+++ b/app/src/components/Upload.js
@@ -9,9 +9,10 @@ import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import {Form as SForm} from "semantic-ui-react";
export function Upload({disabled}) {
- const {setFiles, progresses, destination, setDestination, doClear} = useUploadFile();
const {t} = React.useContext(ThemeContext);
+ const {setFiles, progresses, destination, setDestination, doClear, tagsSelector} = useUploadFile();
+
const onDrop = React.useCallback(async (acceptedFiles) => {
if (acceptedFiles && acceptedFiles.length > 0) {
setFiles(acceptedFiles);
@@ -70,6 +71,7 @@ export function Upload({disabled}) {
+ {tagsSelector}
diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js
index 7e47eddb..c306695c 100644
--- a/app/src/hooks/customHooks.js
+++ b/app/src/hooks/customHooks.js
@@ -30,6 +30,7 @@ import {QueryContext, SettingsContext, StatusContext} from "../contexts/contexts
import {toast} from "react-semantic-toasts-2";
import {useSearch} from "../components/Search";
import _ from "lodash";
+import {TagsSelector} from "../Tags";
const calculatePage = (offset, limit) => {
return offset && limit ? Math.round((offset / limit) + 1) : 1;
@@ -997,6 +998,7 @@ export const useUploadFile = () => {
const [files, setFiles] = useState([]);
const [progresses, setProgresses] = useState({});
const [destination, setDestination] = useState('');
+ const [tagNames, setTagNames] = React.useState([]);
const handleProgress = (name, chunk, totalChunks, status, type) => {
const percent = Math.round((100 * chunk) / totalChunks);
@@ -1031,8 +1033,11 @@ export const useUploadFile = () => {
// Send the size that we're actually sending.
formData.append('chunkSize', chunk.size.toString());
formData.append('chunk', chunk);
+ for (let i = 0; i < tagNames.length; i++) {
+ formData.append('tagNames', tagNames[i]);
+ }
- console.debug(`file upload: tries=${tries} chunkNum=${chunkNum} totalChunks=${totalChunks} chunkSize=${chunk.size} destination=${destination}`);
+ console.debug(`file upload: tries=${tries} chunkNum=${chunkNum} totalChunks=${totalChunks} chunkSize=${chunk.size} destination=${destination} tagNames=${tagNames}`);
const xhr = new XMLHttpRequest();
xhr.open('POST', '/api/files/upload', true);
@@ -1101,7 +1106,27 @@ export const useUploadFile = () => {
doUpload()
}, [JSON.stringify(files)]);
- return {files, setFiles: handleFilesChange, progresses, destination, setDestination, doClear, doUpload}
+ const handleAddTag = (tagName) => {
+ setTagNames([...tagNames, tagName]);
+ }
+
+ const handleUntag = (tagName) => {
+ setTagNames(tagNames.filter(i => i !== tagName));
+ }
+
+ const tagsSelector = ;
+
+ return {
+ destination,
+ doClear,
+ doUpload,
+ files,
+ progresses,
+ setDestination,
+ setFiles: handleFilesChange,
+ tagNames,
+ tagsSelector,
+ }
}
export const useSearchZim = (searchStr, zimId, active, activeTags, defaultLimit = 10) => {
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index ffb3c8fe..3a272d57 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -489,8 +489,7 @@ def link_channel_and_downloads(session: Session):
# Get any Downloads for a Channel's RSS feed.
rss_url = channel.get_rss_url()
- download = downloads_by_url.get(rss_url) if rss_url else None
- if download:
+ if rss_url and (download := downloads_by_url.get(rss_url)):
download.channel_id = channel.id
# Associate any Download which shares a Channel's URL.
diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py
index aa6c5010..0fb31038 100644
--- a/wrolpi/conftest.py
+++ b/wrolpi/conftest.py
@@ -719,6 +719,9 @@ def make_multipart_form():
name, value, filename = form['name'], form['value'], form.get('filename')
if filename:
new_forms.append(f'Content-Disposition: form-data; name="{name}"; filename="{filename}"\n\n{value}\n')
+ elif isinstance(value, list):
+ for part in value:
+ new_forms.append(f'Content-Disposition: form-data; name="{name}"\n\n{part}\n')
else:
new_forms.append(f'Content-Disposition: form-data; name="{name}"\n\n{value}\n')
body = f'\n{boundary}'.join(new_forms)
diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py
index b129554e..841f81e2 100644
--- a/wrolpi/files/api.py
+++ b/wrolpi/files/api.py
@@ -13,6 +13,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileC
from . import lib, schema
from ..api_utils import json_response, api_app
from ..schema import JSONErrorResponse
+from ..tags import Tag
from ..vars import PYTEST
files_bp = Blueprint('Files', '/api/files')
@@ -319,6 +320,15 @@ async def post_upload(request: Request):
except Exception:
raise FileUploadFailed('totalChunks integer is required')
+ tag_names = request.form.getlist('tagNames')
+ if tag_names:
+ if not isinstance(tag_names, list):
+ raise FileUploadFailed('tag_names must be a list')
+ tag_names = [str(i) for i in tag_names]
+ for tag_name in tag_names:
+ if not Tag.find_by_name(tag_name):
+ raise FileUploadFailed(f'Tag does not exist: {tag_name}')
+
filename = pathlib.Path(filename.lstrip('/'))
output = destination / filename
output_str = str(output)
@@ -372,7 +382,8 @@ async def post_upload(request: Request):
if chunk_num == total_chunks:
# File upload is complete.
del api_app.shared_ctx.uploaded_files[output_str]
- background_task(lib.refresh_files([output.parent]))
+ # Upsert this new file (and any related files) into the DB.
+ background_task(lib.upsert_file(output, tag_names=tag_names))
return response.empty(HTTPStatus.CREATED)
# Request the next chunk.
diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py
index c6ffc5cd..5d56bc56 100644
--- a/wrolpi/files/lib.py
+++ b/wrolpi/files/lib.py
@@ -11,7 +11,7 @@ import shutil
import subprocess
from itertools import zip_longest
from pathlib import Path
-from typing import List, Tuple, Union, Dict, Generator
+from typing import List, Tuple, Union, Dict, Generator, Iterable
import cachetools.func
import psycopg2
@@ -319,7 +319,7 @@ def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]:
return files
-def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> pathlib.Path:
+def get_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]]) -> pathlib.Path:
"""Given a list of files, return the file that we can model or index.
@raise NoPrimaryFile: If not primary file can be found."""
@@ -450,6 +450,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]:
+ """Return a new list which does not contain any file paths that are in ignored directories."""
ignored_directories = list(map(str, get_wrolpi_config().ignored_directories))
files = [i for i in files if not any(str(i).startswith(j) for j in ignored_directories)]
return files
@@ -1370,3 +1371,25 @@ def remove_ignored_directory(directory: Union[pathlib.Path, str]):
get_wrolpi_config().ignored_directories = ignored_directories
else:
raise UnknownDirectory('Directory is not ignored')
+
+
+@optional_session(commit=True)
+async def upsert_file(file: pathlib.Path | str, session: Session = None, tag_names: List[str] = None):
+ """Insert/update all files in the provided file's FileGroup."""
+ # Update/Insert all files in the FileGroup.
+ paths = glob_shared_stem(pathlib.Path(file))
+ # Remove any ignored files.
+ paths = remove_files_in_ignored_directories(paths)
+ if paths:
+ file_group = FileGroup.from_paths(session, *paths)
+ # Re-index the contents of the file.
+ session.add(file_group)
+ session.flush([file_group, ])
+ file_group.do_index()
+
+ for tag_name in (tag_names or []):
+ if tag_name not in file_group.tag_names:
+ tag = Tag.find_by_name(tag_name)
+ file_group.add_tag(tag)
+ else:
+ logger.warning('upsert_file called, but all files are in ignored directories!')
diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py
index 6a134cf9..a77c26e6 100644
--- a/wrolpi/files/test/test_api.py
+++ b/wrolpi/files/test/test_api.py
@@ -425,9 +425,10 @@ def test_post_upload_directory(test_session, test_client, test_directory, make_f
assert test_session.query(FileGroup).count() == 1
-def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form):
+def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form, tag_factory):
"""A file can be uploaded in chunks directly to the destination."""
make_files_structure(['uploads/'])
+ tag1, tag2 = tag_factory(), tag_factory()
forms = [
dict(name='chunkNumber', value='0'),
@@ -435,7 +436,8 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value='3'),
- dict(name='chunk', value='foo', filename='chunk')
+ dict(name='chunk', value='foo', filename='chunk'),
+ dict(name='tagNames', value=[tag1.name, tag2.name]),
]
body = make_multipart_form(forms)
request, response = test_client.post('/api/files/upload', content=body,
@@ -453,7 +455,8 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value='3'),
- dict(name='chunk', value='bar', filename='chunk')
+ dict(name='chunk', value='bar', filename='chunk'),
+ dict(name='tagNames', value=[tag1.name, tag2.name]),
]
body = make_multipart_form(forms)
request, response = test_client.post('/api/files/upload', content=body,
@@ -465,7 +468,9 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
assert (test_directory / 'uploads/foo.txt').is_file()
assert (test_directory / 'uploads/foo.txt').read_text() == 'foobar'
- assert test_session.query(FileGroup).count() == 1
+ file_group: FileGroup = test_session.query(FileGroup).one()
+ assert set(file_group.tag_names) == {tag1.name, tag2.name}, 'Two tags should be applied.'
+ assert file_group.indexed, 'File should be indexed after upload.'
@pytest.mark.asyncio