diff --git a/app/src/components/Upload.js b/app/src/components/Upload.js index 9875213e..de98508d 100644 --- a/app/src/components/Upload.js +++ b/app/src/components/Upload.js @@ -1,7 +1,7 @@ import React from "react"; import {useDropzone} from "react-dropzone"; import {Form, Header, Icon, Progress, Segment} from "./Theme"; -import {DirectorySearch, mimetypeIconName} from "./Common"; +import {DirectorySearch, mimetypeIconName, Toggle} from "./Common"; import {useUploadFile} from "../hooks/customHooks"; import _ from "lodash"; import {ThemeContext} from "../contexts/contexts"; @@ -11,7 +11,16 @@ import {Form as SForm} from "semantic-ui-react"; export function Upload({disabled}) { const {t} = React.useContext(ThemeContext); - const {setFiles, progresses, destination, setDestination, doClear, tagsSelector} = useUploadFile(); + const { + setFiles, + progresses, + destination, + setDestination, + doClear, + tagsSelector, + overwrite, + setOverwrite + } = useUploadFile(); const onDrop = React.useCallback(async (acceptedFiles) => { if (acceptedFiles && acceptedFiles.length > 0) { @@ -73,6 +82,8 @@ export function Upload({disabled}) { /> {tagsSelector} + + setOverwrite(!overwrite)}/> {destination ? diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 0898f771..acc38b25 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -1153,6 +1153,7 @@ export const useUploadFile = () => { const [progresses, setProgresses] = useState({}); const [destination, setDestination] = useState(''); const [tagNames, setTagNames] = React.useState([]); + const [overwrite, setOverwrite] = React.useState(false); const handleProgress = (name, chunk, totalChunks, status, type) => { const percent = Math.round((100 * chunk) / totalChunks); @@ -1184,6 +1185,9 @@ export const useUploadFile = () => { formData.append('filename', file.path); formData.append('totalChunks', totalChunks.toString()); formData.append('destination', destination); + if (overwrite) { + formData.append('overwrite', 'true'); + } // Send the size that we're actually sending. formData.append('chunkSize', chunk.size.toString()); formData.append('chunk', chunk); @@ -1262,15 +1266,15 @@ export const useUploadFile = () => { const tagsSelector = setTagNames(i)}/>; return { - destination, + destination, setDestination, doClear, doUpload, files, progresses, - setDestination, setFiles: handleFilesChange, tagNames, tagsSelector, + overwrite, setOverwrite, } } diff --git a/modules/archive/api.py b/modules/archive/api.py index 240900ce..9e57a6df 100644 --- a/modules/archive/api.py +++ b/modules/archive/api.py @@ -95,7 +95,7 @@ async def singlefile_upload_switch_handler(singlefile): Events.send_archive_uploaded(f'Created Archive from upload: {name}', url=archive.location) url = archive.file_group.url if url and (download := Download.get_by_url(url)): - if download.is_failed and download.downloader == ArchiveDownloader.name: + if (download.is_failed or download.is_deferred) and download.downloader == ArchiveDownloader.name: # Download was attempted and failed, user manually archived the URL. download_manager.delete_download(download.id) diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index 8742e1ef..1650d2ac 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -342,6 +342,16 @@ async def post_upload(request: Request): except Exception: raise FileUploadFailed('totalChunks integer is required') + try: + overwrite = request.form['overwrite'][0] + if overwrite.lower().strip() == 'true': + overwrite = True + else: + overwrite = False + except Exception: + overwrite = False + + print(f'{overwrite=} {type(overwrite)=}') tag_names = request.form.getlist('tagNames') if tag_names: if not isinstance(tag_names, list): @@ -375,13 +385,23 @@ async def post_upload(request: Request): # Respond with a request for the correct chunk number. return json_response({'expected_chunk': expected_chunk_num}, HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE) - if chunk_num == 0 and output.is_file(): - # Do not overwrite files that already exist. + if chunk_num == 0 and output.is_file() and not overwrite: + # Do not overwrite files that already exist, unless explicitly requested. raise FileUploadFailed('File already exists!') if output.is_dir(): raise FileUploadFailed('File already exists as a directory!') if chunk_num == 0: + try: + # Delete any conflicting FileGroups if the user is overwriting. + await lib.delete(output) + except InvalidFile: + # No conflicting files, good. + pass + except Exception as e: + logger.error('Failed to delete conflicting upload file', exc_info=e) + raise + try: output.touch() except FileNotFoundError: diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index 54538e58..b91ea546 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -549,8 +549,7 @@ def test_post_upload_directory(test_session, test_client, test_directory, make_f @pytest.mark.asyncio async def test_post_upload(test_session, async_client, test_directory, make_files_structure, make_multipart_form, - tag_factory, - video_bytes): + tag_factory, video_bytes): """A file can be uploaded in chunks directly to the destination.""" make_files_structure(['uploads/']) tag1, tag2 = await tag_factory(), await tag_factory() @@ -606,6 +605,77 @@ async def test_post_upload(test_session, async_client, test_directory, make_file assert not video.info_json_path assert not video.channel and not video.channel_id + # Can't upload again because the file already exists. + forms = [ + dict(name='chunkNumber', value='0'), + dict(name='filename', value='video.mp4'), + dict(name='totalChunks', value='1'), + dict(name='destination', value='uploads'), + dict(name='chunkSize', value=1_000_000), + dict(name='chunk', value=part1, filename='chunk'), + ] + body = make_multipart_form(forms) + headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'} + request, response = await async_client.post('/api/files/upload', content=body, headers=headers) + assert response.status_code == HTTPStatus.BAD_REQUEST, response.content.decode() + assert response.json['error'] == 'File already exists!' + + # FileGroup was not deleted. + file_group: FileGroup = test_session.query(FileGroup).one() + assert file_group + + # Remove Tag so file can be uploaded again. + file_group.set_tags([], test_session) + + # An existing file can be overwritten. + forms = [ + dict(name='chunkNumber', value='0'), + dict(name='filename', value='video.mp4'), + dict(name='totalChunks', value='1'), + dict(name='destination', value='uploads'), + dict(name='chunkSize', value=1_000_000), + dict(name='chunk', value=part1, filename='chunk'), + dict(name='overwrite', value=True), + ] + body = make_multipart_form(forms) + headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'} + request, response = await async_client.post('/api/files/upload', content=body, headers=headers) + assert response.status_code == HTTPStatus.OK, response.content.decode() + + file_group: FileGroup = test_session.query(FileGroup).one_or_none() + assert not file_group, 'Old FileGroup should be deleted when new upload starts.' + + # Upload the second part of the file again. + forms = [ + dict(name='chunkNumber', value='1'), + dict(name='filename', value='video.mp4'), + dict(name='totalChunks', value='1'), + dict(name='destination', value='uploads'), + dict(name='chunkSize', value=56318), + dict(name='chunk', value=part2, filename='chunk'), + ] + body = make_multipart_form(forms) + request, response = await async_client.post('/api/files/upload', content=body, headers=headers) + assert response.status_code == HTTPStatus.CREATED, response.content.decode() + + # Make the same checks as those done above because the file was uploaded again. + assert output.is_file() + assert get_mimetype(output) == 'video/mp4' + assert output.stat().st_size == len(video_bytes) + assert output.read_bytes() == video_bytes + assert hashlib.md5(output.read_bytes()).hexdigest() == '2738c53bd7c01b01d408da11a55bfa36' + + file_group: FileGroup = test_session.query(FileGroup).one() + assert file_group.mimetype == 'video/mp4' + assert not file_group.tag_names, 'No tags this time.' + assert file_group.indexed, 'File should be indexed after upload.' + assert file_group.model, 'File should be modeled' + # Video was modeled, but has no Channel. + video: Video = test_session.query(Video).one() + assert video.video_path + assert not video.info_json_path + assert not video.channel and not video.channel_id + def test_post_upload_text(test_session, test_client, test_directory, make_files_structure, make_multipart_form): """A file that cannot be modeled can still be uploaded, and is indexed."""