Adding feature so that tags can be applied when uploading files.

This commit is contained in:
Roland 2024-07-21 15:04:56 -06:00
parent 63738bd70f
commit f3439c2bd5
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
7 changed files with 80 additions and 12 deletions

View file

@ -9,9 +9,10 @@ import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import {Form as SForm} from "semantic-ui-react";
export function Upload({disabled}) {
const {setFiles, progresses, destination, setDestination, doClear} = useUploadFile();
const {t} = React.useContext(ThemeContext);
const {setFiles, progresses, destination, setDestination, doClear, tagsSelector} = useUploadFile();
const onDrop = React.useCallback(async (acceptedFiles) => {
if (acceptedFiles && acceptedFiles.length > 0) {
setFiles(acceptedFiles);
@ -70,6 +71,7 @@ export function Upload({disabled}) {
<SForm.Field required>
<label>Destination</label>
<DirectorySearch onSelect={handleDestination} disabled={disabled}/>
{tagsSelector}
</SForm.Field>
</Form>

View file

@ -30,6 +30,7 @@ import {QueryContext, SettingsContext, StatusContext} from "../contexts/contexts
import {toast} from "react-semantic-toasts-2";
import {useSearch} from "../components/Search";
import _ from "lodash";
import {TagsSelector} from "../Tags";
const calculatePage = (offset, limit) => {
return offset && limit ? Math.round((offset / limit) + 1) : 1;
@ -997,6 +998,7 @@ export const useUploadFile = () => {
const [files, setFiles] = useState([]);
const [progresses, setProgresses] = useState({});
const [destination, setDestination] = useState('');
const [tagNames, setTagNames] = React.useState([]);
const handleProgress = (name, chunk, totalChunks, status, type) => {
const percent = Math.round((100 * chunk) / totalChunks);
@ -1031,8 +1033,11 @@ export const useUploadFile = () => {
// Send the size that we're actually sending.
formData.append('chunkSize', chunk.size.toString());
formData.append('chunk', chunk);
for (let i = 0; i < tagNames.length; i++) {
formData.append('tagNames', tagNames[i]);
}
console.debug(`file upload: tries=${tries} chunkNum=${chunkNum} totalChunks=${totalChunks} chunkSize=${chunk.size} destination=${destination}`);
console.debug(`file upload: tries=${tries} chunkNum=${chunkNum} totalChunks=${totalChunks} chunkSize=${chunk.size} destination=${destination} tagNames=${tagNames}`);
const xhr = new XMLHttpRequest();
xhr.open('POST', '/api/files/upload', true);
@ -1101,7 +1106,27 @@ export const useUploadFile = () => {
doUpload()
}, [JSON.stringify(files)]);
return {files, setFiles: handleFilesChange, progresses, destination, setDestination, doClear, doUpload}
const handleAddTag = (tagName) => {
setTagNames([...tagNames, tagName]);
}
const handleUntag = (tagName) => {
setTagNames(tagNames.filter(i => i !== tagName));
}
const tagsSelector = <TagsSelector selectedTagNames={tagNames} onAdd={handleAddTag} onRemove={handleUntag}/>;
return {
destination,
doClear,
doUpload,
files,
progresses,
setDestination,
setFiles: handleFilesChange,
tagNames,
tagsSelector,
}
}
export const useSearchZim = (searchStr, zimId, active, activeTags, defaultLimit = 10) => {

View file

@ -489,8 +489,7 @@ def link_channel_and_downloads(session: Session):
# Get any Downloads for a Channel's RSS feed.
rss_url = channel.get_rss_url()
download = downloads_by_url.get(rss_url) if rss_url else None
if download:
if rss_url and (download := downloads_by_url.get(rss_url)):
download.channel_id = channel.id
# Associate any Download which shares a Channel's URL.

View file

@ -719,6 +719,9 @@ def make_multipart_form():
name, value, filename = form['name'], form['value'], form.get('filename')
if filename:
new_forms.append(f'Content-Disposition: form-data; name="{name}"; filename="{filename}"\n\n{value}\n')
elif isinstance(value, list):
for part in value:
new_forms.append(f'Content-Disposition: form-data; name="{name}"\n\n{part}\n')
else:
new_forms.append(f'Content-Disposition: form-data; name="{name}"\n\n{value}\n')
body = f'\n{boundary}'.join(new_forms)

View file

@ -13,6 +13,7 @@ from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileC
from . import lib, schema
from ..api_utils import json_response, api_app
from ..schema import JSONErrorResponse
from ..tags import Tag
from ..vars import PYTEST
files_bp = Blueprint('Files', '/api/files')
@ -319,6 +320,15 @@ async def post_upload(request: Request):
except Exception:
raise FileUploadFailed('totalChunks integer is required')
tag_names = request.form.getlist('tagNames')
if tag_names:
if not isinstance(tag_names, list):
raise FileUploadFailed('tag_names must be a list')
tag_names = [str(i) for i in tag_names]
for tag_name in tag_names:
if not Tag.find_by_name(tag_name):
raise FileUploadFailed(f'Tag does not exist: {tag_name}')
filename = pathlib.Path(filename.lstrip('/'))
output = destination / filename
output_str = str(output)
@ -372,7 +382,8 @@ async def post_upload(request: Request):
if chunk_num == total_chunks:
# File upload is complete.
del api_app.shared_ctx.uploaded_files[output_str]
background_task(lib.refresh_files([output.parent]))
# Upsert this new file (and any related files) into the DB.
background_task(lib.upsert_file(output, tag_names=tag_names))
return response.empty(HTTPStatus.CREATED)
# Request the next chunk.

View file

@ -11,7 +11,7 @@ import shutil
import subprocess
from itertools import zip_longest
from pathlib import Path
from typing import List, Tuple, Union, Dict, Generator
from typing import List, Tuple, Union, Dict, Generator, Iterable
import cachetools.func
import psycopg2
@ -319,7 +319,7 @@ def _paths_to_files_dict(group: List[pathlib.Path]) -> List[dict]:
return files
def get_primary_file(files: Union[Tuple[pathlib.Path], List[pathlib.Path]]) -> pathlib.Path:
def get_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]]) -> pathlib.Path:
"""Given a list of files, return the file that we can model or index.
@raise NoPrimaryFile: If not primary file can be found."""
@ -450,6 +450,7 @@ def _upsert_files(files: List[pathlib.Path], idempotency: datetime.datetime):
def remove_files_in_ignored_directories(files: List[pathlib.Path]) -> List[pathlib.Path]:
"""Return a new list which does not contain any file paths that are in ignored directories."""
ignored_directories = list(map(str, get_wrolpi_config().ignored_directories))
files = [i for i in files if not any(str(i).startswith(j) for j in ignored_directories)]
return files
@ -1370,3 +1371,25 @@ def remove_ignored_directory(directory: Union[pathlib.Path, str]):
get_wrolpi_config().ignored_directories = ignored_directories
else:
raise UnknownDirectory('Directory is not ignored')
@optional_session(commit=True)
async def upsert_file(file: pathlib.Path | str, session: Session = None, tag_names: List[str] = None):
"""Insert/update all files in the provided file's FileGroup."""
# Update/Insert all files in the FileGroup.
paths = glob_shared_stem(pathlib.Path(file))
# Remove any ignored files.
paths = remove_files_in_ignored_directories(paths)
if paths:
file_group = FileGroup.from_paths(session, *paths)
# Re-index the contents of the file.
session.add(file_group)
session.flush([file_group, ])
file_group.do_index()
for tag_name in (tag_names or []):
if tag_name not in file_group.tag_names:
tag = Tag.find_by_name(tag_name)
file_group.add_tag(tag)
else:
logger.warning('upsert_file called, but all files are in ignored directories!')

View file

@ -425,9 +425,10 @@ def test_post_upload_directory(test_session, test_client, test_directory, make_f
assert test_session.query(FileGroup).count() == 1
def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form):
def test_post_upload(test_session, test_client, test_directory, make_files_structure, make_multipart_form, tag_factory):
"""A file can be uploaded in chunks directly to the destination."""
make_files_structure(['uploads/'])
tag1, tag2 = tag_factory(), tag_factory()
forms = [
dict(name='chunkNumber', value='0'),
@ -435,7 +436,8 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value='3'),
dict(name='chunk', value='foo', filename='chunk')
dict(name='chunk', value='foo', filename='chunk'),
dict(name='tagNames', value=[tag1.name, tag2.name]),
]
body = make_multipart_form(forms)
request, response = test_client.post('/api/files/upload', content=body,
@ -453,7 +455,8 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value='3'),
dict(name='chunk', value='bar', filename='chunk')
dict(name='chunk', value='bar', filename='chunk'),
dict(name='tagNames', value=[tag1.name, tag2.name]),
]
body = make_multipart_form(forms)
request, response = test_client.post('/api/files/upload', content=body,
@ -465,7 +468,9 @@ def test_post_upload(test_session, test_client, test_directory, make_files_struc
assert (test_directory / 'uploads/foo.txt').is_file()
assert (test_directory / 'uploads/foo.txt').read_text() == 'foobar'
assert test_session.query(FileGroup).count() == 1
file_group: FileGroup = test_session.query(FileGroup).one()
assert set(file_group.tag_names) == {tag1.name, tag2.name}, 'Two tags should be applied.'
assert file_group.indexed, 'File should be indexed after upload.'
@pytest.mark.asyncio