diff --git a/app/package.json b/app/package.json index 103b9680..829604d4 100644 --- a/app/package.json +++ b/app/package.json @@ -10,6 +10,7 @@ "react": "^16.13.1", "react-dom": "^16.13.1", "react-dropzone": "^11.4.2", + "react-keyed-file-browser": "^1.14.0", "react-router-dom": "^5.2.0", "react-scripts": "^3.4.1", "react-semantic-toasts": "^0.6.5", diff --git a/app/src/App.js b/app/src/App.js index 274bed5e..3c7f4f5c 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -15,6 +15,7 @@ import {Saver} from "./components/Upload"; import {useSearchParam} from "./hooks/useSearchParam"; import {searchArchives, searchVideos} from "./api"; import {MoreButton, SearchInput} from "./components/Common"; +import {FilesRoute} from "./components/Files"; const useSearch = () => { let [searchStr, setSearchStr] = useSearchParam('q'); @@ -107,6 +108,7 @@ export default function App() { + diff --git a/app/src/api.js b/app/src/api.js index 778245fd..5abbb22f 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -374,3 +374,14 @@ export async function getDownloaders() { let response = await apiGet(`${API_URI}/downloaders`); return await response.json(); } + +export async function getFiles(directories) { + let body = {directories: directories || []}; + let response = await apiPost(`${API_URI}/files`, body); + return await response.json(); +} + +export async function deleteFile(file) { + let body = {file: file}; + await apiPost(`${API_URI}/files/delete`, body); +} diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 29877e52..e50fae2f 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -90,6 +90,12 @@ export function uploadDate(d) { return upload_date; } +export function secondsToDateTime(seconds) { + var t = new Date(1970, 0, 1); // Epoch + t.setSeconds(seconds); + return t; +} + export function VideoCard({video}) { let video_url = `/videos/video/${video.id}`; let upload_date = uploadDate(video.upload_date); diff --git a/app/src/components/Files.js b/app/src/components/Files.js new file mode 100644 index 00000000..d39226f6 --- /dev/null +++ b/app/src/components/Files.js @@ -0,0 +1,119 @@ +import React from "react"; +import FileBrowser from 'react-keyed-file-browser'; +import {Button, Confirm, Header, Icon, Modal} from "semantic-ui-react"; +import 'react-keyed-file-browser/dist/react-keyed-file-browser.css'; +import {deleteFile, getFiles} from "../api"; +import {humanFileSize, secondsToDateTime} from "./Common"; + +const icons = { + File: , + Folder: , + FolderOpen: , + Image: , + PDF: , +}; + +class Files extends React.Component { + constructor(props) { + super(props); + this.state = { + deleteOpen: false, + files: [], + openFolders: [], + previewOpen: false, + selectedFile: null, + }; + } + + async componentDidMount() { + await this.fetchFiles(); + } + + fetchFiles = async () => { + let {files} = await getFiles(this.state.openFolders); + for (let i = 0; i < files.length; i++) { + files[i]['modified'] = secondsToDateTime(files[i]['modified']); + } + this.setState({files: files}); + } + + handleFolderChange = async (file, browserProps) => { + let openFolders = Object.keys(browserProps.openFolders); + this.setState({openFolders}, this.fetchFiles); + } + + onDownloadFile = async () => { + // Open file in new tab/window. + window.open(`/media/${this.state.selectedFile.key}`); + } + + onSelectFile = async (file) => { + this.setState({selectedFile: file}, this.openPreview); + } + + onDeleteFile = async () => { + await deleteFile(this.state.selectedFile.key); + await this.fetchFiles(); + this.closePreview(); + } + + openPreview = () => { + this.setState({previewOpen: true}); + } + + closePreview = () => { + this.setState({previewOpen: false, selectedFile: null, deleteOpen: false}); + } + + openDelete = () => { + this.setState({deleteOpen: true}); + } + + closeDelete = () => { + this.setState({deleteOpen: false}); + } + + render() { + return <> + {this.state.selectedFile && + + File Preview + +
{this.state.selectedFile.name}
+

{humanFileSize(this.state.selectedFile.size)}

+ + + + + + + } + <>} // Hide the preview that the 3rd party provided. + /> + + } +} + +export class FilesRoute extends React.Component { + render() { + return ( + + ); + } +} diff --git a/app/src/components/Nav.js b/app/src/components/Nav.js index 7142ffd0..c93d8877 100644 --- a/app/src/components/Nav.js +++ b/app/src/components/Nav.js @@ -15,6 +15,7 @@ const links = [ }, {text: 'Inventory', to: '/inventory', exact: true}, {text: 'Archive', to: '/archive', exact: true}, + {text: 'Files', to: '/files', exact: true}, { text: 'Apps', links: [ {to: '/apps/otp', text: 'One Time Pad', exact: true}, diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 4d77aef4..d1ac93ac 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -8,7 +8,7 @@ ENV PATH /app/node_modules/.bin:$PATH COPY app/public /app/public COPY app/src /app/src COPY app/package.json /app/ -RUN rm -rf /app/node_modules +RUN rm -rf /app/node_modules /app/package-lock.json # Increase network timeout because a Raspberry Pi is slow. RUN yarn --network-timeout 100000 diff --git a/modules/archive/lib.py b/modules/archive/lib.py index adc3177d..6f8921ff 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -15,7 +15,7 @@ from sqlalchemy.orm import Session from modules.archive.models import Domain, Archive from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir -from wrolpi.dates import now, strptime_ms, strftime_ms +from wrolpi.dates import now, strptime_ms, strftime_ms, Seconds from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive from wrolpi.vars import DOCKERIZED, PYTEST @@ -69,7 +69,7 @@ def get_new_archive_files(url: str): return ret -ARCHIVE_TIMEOUT = 10 * 60 # Wait at most 10 minutes for response. +ARCHIVE_TIMEOUT = Seconds.minute * 10 # Wait at most 10 minutes for response. def request_archive(url: str): diff --git a/modules/files/__init__.py b/modules/files/__init__.py new file mode 100644 index 00000000..03476b2d --- /dev/null +++ b/modules/files/__init__.py @@ -0,0 +1 @@ +from .api import bp diff --git a/modules/files/api.py b/modules/files/api.py new file mode 100644 index 00000000..72cee406 --- /dev/null +++ b/modules/files/api.py @@ -0,0 +1,66 @@ +import pathlib +from typing import List +from urllib.request import Request + +from sanic import response + +from wrolpi.common import get_media_directory +from wrolpi.errors import InvalidFile +from wrolpi.root_api import get_blueprint, json_response +from wrolpi.schema import validate_doc +from . import lib, schema + +bp = get_blueprint('Files', '/api/files') + + +def paths_to_files(paths: List[pathlib.Path]): + media_directory = get_media_directory() + new_files = [] + for path in paths: + stat = path.stat() + key = path.relative_to(media_directory) + modified = stat.st_mtime + if path.is_dir(): + key = f'{key}/' + new_files.append(dict( + key=key, + modified=modified, + url=key, + name=path.name, + )) + else: + # A File should know it's size. + new_files.append(dict( + key=key, + modified=modified, + size=stat.st_size, + url=key, + name=path.name, + )) + return new_files + + +@bp.post('/') +@validate_doc( + 'List files in a directory', + consumes=schema.FilesRequest, +) +async def get_files(_: Request, data: dict): + directories = data.get('directories') or [] + + files = lib.list_files(directories) + files = paths_to_files(files) + return json_response({'files': files}) + + +@bp.post('/delete') +@validate_doc( + 'Delete a single file. Returns an error if WROL Mode is enabled.', + consumes=schema.DeleteRequest, +) +async def delete_file(_: Request, data: dict): + file = data.get('file') + if not file: + raise InvalidFile('file cannot be empty') + lib.delete_file(file) + return response.empty() diff --git a/modules/files/conftest.py b/modules/files/conftest.py new file mode 100644 index 00000000..5c241a93 --- /dev/null +++ b/modules/files/conftest.py @@ -0,0 +1,21 @@ +import pathlib +from typing import List + +import pytest + + +@pytest.fixture +def make_files_structure(test_directory): + def create_files(paths: List[str]) -> List[pathlib.Path]: + files = [] + for name in paths: + path = test_directory / name + if name.endswith('/'): + path.mkdir() + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + files.append(path) + return files + + return create_files diff --git a/modules/files/lib.py b/modules/files/lib.py new file mode 100644 index 00000000..438f17c6 --- /dev/null +++ b/modules/files/lib.py @@ -0,0 +1,60 @@ +from pathlib import Path +from typing import List + +from wrolpi.common import get_media_directory, wrol_mode_check +from wrolpi.errors import InvalidFile + + +def filter_parent_directories(directories: List[Path]) -> List[Path]: + """ + Remove parent directories if their children are in the list. + + >>> filter_parent_directories([Path('foo'), Path('foo/bar'), Path('baz')]) + [Path('foo/bar'), Path('baz')] + """ + unique_children = set() + for directory in sorted(directories): + for parent in directory.parents: + # Remove any parent of this child. + if parent in unique_children: + unique_children.remove(parent) + unique_children.add(directory) + + # Restore the original order. + new_directories = [i for i in directories if i in unique_children] + return new_directories + + +def list_files(directories: List[str]) -> List[Path]: + """ + List all files down to the directories provided. This includes all parent directories of the directories. + """ + media_directory = get_media_directory() + + # Always display the media_directory files. + paths = list(media_directory.iterdir()) + + if directories: + directories = [media_directory / i for i in directories if i] + directories = filter_parent_directories(directories) + for directory in directories: + directory = media_directory / directory + for parent in directory.parents: + is_relative_to = str(media_directory).startswith(str(parent)) + if parent == Path('.') or is_relative_to: + continue + paths.extend(parent.iterdir()) + paths.extend(directory.iterdir()) + + return paths + + +@wrol_mode_check +def delete_file(file: str): + """ + Delete a file in the media directory. + """ + file = get_media_directory() / file + if file.is_dir() or not file.is_file(): + raise InvalidFile(f'Invalid file {file}') + file.unlink() diff --git a/modules/files/schema.py b/modules/files/schema.py new file mode 100644 index 00000000..6b82bb47 --- /dev/null +++ b/modules/files/schema.py @@ -0,0 +1,9 @@ +from sanic_openapi import doc + + +class FilesRequest: + directories = doc.List(doc.String()) + + +class DeleteRequest: + file = doc.String() diff --git a/modules/files/test/__init__.py b/modules/files/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/files/test/test_api.py b/modules/files/test/test_api.py new file mode 100644 index 00000000..0fb1ea41 --- /dev/null +++ b/modules/files/test/test_api.py @@ -0,0 +1,116 @@ +import json +from http import HTTPStatus +from itertools import zip_longest +from unittest import mock + +import pytest + +from wrolpi.errors import API_ERRORS, WROLModeEnabled + + +def test_list_files_api(test_client, make_files_structure, test_directory): + files = [ + 'archives/bar.txt', + 'archives/baz/bar.txt', + 'archives/baz/foo.txt', + 'archives/foo.txt', + 'empty directory/', + 'videos/other video.mp4', + 'videos/some video.mp4', + ] + files = make_files_structure(files) + files[0].write_text('bar contents') + + def check_get_files(directories, expected_files): + request, response = test_client.post('/api/files', content=json.dumps({'directories': directories})) + assert not response.json.get('errors') + results = sorted(response.json['files'], key=lambda i: i['key']) + for f1, f2 in zip_longest(results, expected_files): + if f1 is None or f2 is None: + assert f1 == f2 + for key in f2.keys(): + assert f1[key] == f2[key], f'{f1} != {f2}' + + # Requesting no directories results in the top-level results. + expected = [dict(key='archives/'), dict(key='empty directory/'), dict(key='videos/')] + check_get_files([], expected) + # empty directory is empty + check_get_files(['empty directory'], expected) + + expected = [ + dict(key='archives/'), + dict(key='archives/bar.txt', size=12), + dict(key='archives/baz/'), + dict(key='archives/foo.txt'), + dict(key='empty directory/'), + dict(key='videos/'), + ] + check_get_files(['archives'], expected) + + # Sub-directories are supported. + expected = [ + dict(key='archives/'), + dict(key='archives/bar.txt', size=12), + dict(key='archives/baz/'), + dict(key='archives/baz/bar.txt'), + dict(key='archives/baz/foo.txt'), + dict(key='archives/foo.txt'), + dict(key='empty directory/'), + dict(key='videos/'), + ] + check_get_files(['archives/baz'], expected) + + expected = [ + dict(key='archives/'), + dict(key='archives/bar.txt', size=12), + dict(key='archives/baz/'), + dict(key='archives/foo.txt'), + dict(key='empty directory/'), + dict(key='videos/'), + dict(key='videos/other video.mp4'), + dict(key='videos/some video.mp4'), + ] + check_get_files(['archives', 'videos'], expected) + # Order does not matter. + check_get_files(['archives', 'videos', 'empty directory'], expected) + check_get_files(['archives', 'empty directory', 'videos'], expected) + + +def test_delete_file(test_client, make_files_structure, test_directory): + files = ['bar.txt', 'baz/'] + make_files_structure(files) + + request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'bar.txt'})) + assert response.status_code == HTTPStatus.NO_CONTENT + assert not (test_directory / 'bar.txt').is_file() + assert (test_directory / 'baz').is_dir() + + request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'baz'})) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert not (test_directory / 'bar.txt').is_file() + assert (test_directory / 'baz').is_dir() + + request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'bad file'})) + assert response.status_code == HTTPStatus.BAD_REQUEST + + +@pytest.mark.parametrize( + 'file', [ + '', + ] +) +def test_delete_invalid_file(test_client, file): + with mock.patch('modules.files.api.lib.delete_file') as mock_delete_file: + request, response = test_client.post('/api/files/delete', content=json.dumps({'file': file})) + assert response.status_code == HTTPStatus.BAD_REQUEST + mock_delete_file.assert_not_called() + + +def test_delete_wrol_mode(test_client): + """ + Can't delete a file when WROL Mode is enabled. + """ + with mock.patch('wrolpi.common.wrol_mode_enabled', lambda: True): + request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'foo'})) + assert response.status_code == HTTPStatus.FORBIDDEN + assert response.json['code'] == API_ERRORS[WROLModeEnabled]['code'] diff --git a/modules/files/test/test_lib.py b/modules/files/test/test_lib.py new file mode 100644 index 00000000..1a53bb29 --- /dev/null +++ b/modules/files/test/test_lib.py @@ -0,0 +1,124 @@ +from pathlib import Path + +import pytest + +from modules.files import lib +from wrolpi.errors import InvalidFile + + +def test_list_files(make_files_structure, test_directory): + files = [ + 'archives/bar.txt', + 'archives/baz/bar.txt', + 'archives/baz/foo.txt', + 'archives/foo.txt', + 'empty directory/', + 'videos/other video.mp4', + 'videos/some video.mp4', + ] + make_files_structure(files) + + def check_list_files(path, expected): + result = lib.list_files(path) + result = sorted(result) + assert [str(i.relative_to(test_directory)) for i in result] == expected + + check_list_files([], ['archives', 'empty directory', 'videos']) + # Falsey directories are ignored. + check_list_files([None, ''], ['archives', 'empty directory', 'videos']) + check_list_files(['empty directory'], ['archives', 'empty directory', 'videos']) + check_list_files(['empty directory'], ['archives', 'empty directory', 'videos']) + # Trailing slash is ignored. + check_list_files(['empty directory/'], ['archives', 'empty directory', 'videos']) + + check_list_files(['archives'], [ + 'archives', + 'archives/bar.txt', + 'archives/baz', + 'archives/foo.txt', + 'empty directory', + 'videos', + ]) + check_list_files(['archives/baz'], [ + 'archives', + 'archives/bar.txt', + 'archives/baz', + 'archives/baz/bar.txt', + 'archives/baz/foo.txt', + 'archives/foo.txt', + 'empty directory', + 'videos', + ]) + # Including duplicate directories does not change results. + check_list_files(['archives/baz', 'archives'], [ + 'archives', + 'archives/bar.txt', + 'archives/baz', + 'archives/baz/bar.txt', + 'archives/baz/foo.txt', + 'archives/foo.txt', + 'empty directory', + 'videos', + ]) + check_list_files(['videos'], [ + 'archives', + 'empty directory', + 'videos', + 'videos/other video.mp4', + 'videos/some video.mp4', + ]) + # All files and directories are listed. + check_list_files(['videos', 'archives/baz'], [ + 'archives', + 'archives/bar.txt', + 'archives/baz', + 'archives/baz/bar.txt', + 'archives/baz/foo.txt', + 'archives/foo.txt', + 'empty directory', + 'videos', + 'videos/other video.mp4', + 'videos/some video.mp4', + ]) + + +@pytest.mark.parametrize( + 'directories,expected', [ + ([Path('foo')], [Path('foo')]), + ([Path('foo'), Path('bar')], [Path('foo'), Path('bar')]), + ([Path('foo'), Path('foo/bar'), Path('baz')], [Path('foo/bar'), Path('baz')]), + ([Path('foo/bar'), Path('foo'), Path('baz')], [Path('foo/bar'), Path('baz')]), + ([Path('baz'), Path('foo/bar'), Path('foo')], [Path('baz'), Path('foo/bar')]), + ([Path('foo/bar'), Path('foo'), Path('foo/bar/baz')], [Path('foo/bar/baz')]), + ] +) +def test_filter_parent_directories(directories, expected): + assert lib.filter_parent_directories(directories) == expected + + +def test_delete_file(make_files_structure, test_directory): + """ + File in the media directory can be deleted. + """ + files = [ + 'archives/foo.txt', + 'bar.txt', + 'baz/', + ] + make_files_structure(files) + + lib.delete_file('bar.txt') + assert (test_directory / 'archives/foo.txt').is_file() + assert not (test_directory / 'bar.txt').is_file() + assert (test_directory / 'baz').is_dir() + + lib.delete_file('archives/foo.txt') + assert not (test_directory / 'archives/foo.txt').is_file() + assert not (test_directory / 'bar.txt').is_file() + + with pytest.raises(InvalidFile): + lib.delete_file('baz') + with pytest.raises(InvalidFile): + lib.delete_file('does not exist') + + assert (test_directory / 'baz').is_dir() diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 499bd484..f6610a84 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import Session from wrolpi.common import logger, chunks, get_config from wrolpi.common import save_settings_config -from wrolpi.dates import from_timestamp +from wrolpi.dates import from_timestamp, Seconds from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.media_path import MediaPath from wrolpi.vars import PYTEST @@ -21,7 +21,7 @@ from .models import Channel, Video logger = logger.getChild(__name__) -DEFAULT_DOWNLOAD_FREQUENCY = 60 * 60 * 24 * 7 # weekly +DEFAULT_DOWNLOAD_FREQUENCY = Seconds.week def refresh_channel_videos(channel: Channel): diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index c8d72b87..56f1a8ea 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -15,7 +15,7 @@ from sqlalchemy.engine import Engine, create_engine from sqlalchemy.orm import Session, sessionmaker from wrolpi.common import set_test_media_directory, Base, get_example_config -from wrolpi.dates import fake_now # noqa +from wrolpi.dates import set_test_now from wrolpi.db import postgres_engine, get_db_args from wrolpi.downloader import DownloadManager from wrolpi.root_api import BLUEPRINTS, api_app @@ -122,3 +122,11 @@ def test_client() -> SanicTestClient: def test_download_manager(): manager = DownloadManager() return manager + + +@pytest.fixture +def fake_now(): + try: + yield set_test_now + finally: + set_test_now(None) # reset now() to its original functionality. diff --git a/wrolpi/dates.py b/wrolpi/dates.py index 14926e7e..381eea4d 100644 --- a/wrolpi/dates.py +++ b/wrolpi/dates.py @@ -2,7 +2,6 @@ import logging from datetime import datetime, date, timezone from enum import Enum -import pytest import pytz from sqlalchemy import types @@ -20,17 +19,14 @@ class Seconds(int, Enum): hour = minute * 60 day = hour * 24 week = day * 7 + year = day * 366 -@pytest.fixture -def fake_now(): - def _(dt: datetime): - if not dt.tzinfo: - dt = local_timezone(dt) - global TEST_DATETIME - TEST_DATETIME = dt - - return _ +def set_test_now(dt: datetime): + if dt and not dt.tzinfo: + dt = local_timezone(dt) + global TEST_DATETIME + TEST_DATETIME = dt def set_timezone(tz: pytz.timezone): diff --git a/wrolpi/errors.py b/wrolpi/errors.py index 7f6396ef..bccb667f 100644 --- a/wrolpi/errors.py +++ b/wrolpi/errors.py @@ -136,6 +136,10 @@ class UnrecoverableDownloadError(APIError): pass +class InvalidFile(APIError): + pass + + error_codes = iter(range(1, 1000)) API_ERRORS = { @@ -299,4 +303,9 @@ API_ERRORS = { 'message': 'Download experienced an error which cannot be recovered. Download will be deleted.', 'status': HTTPStatus.INTERNAL_SERVER_ERROR, }, + InvalidFile: { + 'code': next(error_codes), + 'message': 'File does not exist or is a directory', + 'status': HTTPStatus.BAD_REQUEST, + } }