Merge pull request #89 from lrnselfreliance/feature/files

Feature/files

Signed-off-by: Roland <roland@learningselfreliance.com>

app/package.json

@@ -10,6 +10,7 @@
     "react": "^16.13.1",
     "react-dom": "^16.13.1",
     "react-dropzone": "^11.4.2",
+    "react-keyed-file-browser": "^1.14.0",
     "react-router-dom": "^5.2.0",
     "react-scripts": "^3.4.1",
     "react-semantic-toasts": "^0.6.5",

app/src/App.js

@@ -15,6 +15,7 @@ import {Saver} from "./components/Upload";
 import {useSearchParam} from "./hooks/useSearchParam";
 import {searchArchives, searchVideos} from "./api";
 import {MoreButton, SearchInput} from "./components/Common";
+import {FilesRoute} from "./components/Files";
 
 const useSearch = () => {
     let [searchStr, setSearchStr] = useSearchParam('q');
@@ -107,6 +108,7 @@ export default function App() {
                     <Route path="/apps" component={AppsRoute}/>
                     <Route path="/inventory" component={InventoryRoute}/>
                     <Route path='/archive' component={ArchiveRoute}/>
+                    <Route path='/files' component={FilesRoute}/>
                     <Route component={PageNotFound}/>
                 </Switch>
             </Container>

app/src/api.js

@@ -374,3 +374,14 @@ export async function getDownloaders() {
     let response = await apiGet(`${API_URI}/downloaders`);
     return await response.json();
 }
+
+export async function getFiles(directories) {
+    let body = {directories: directories || []};
+    let response = await apiPost(`${API_URI}/files`, body);
+    return await response.json();
+}
+
+export async function deleteFile(file) {
+    let body = {file: file};
+    await apiPost(`${API_URI}/files/delete`, body);
+}

app/src/components/Common.js

@@ -90,6 +90,12 @@ export function uploadDate(d) {
     return upload_date;
 }
 
+export function secondsToDateTime(seconds) {
+    var t = new Date(1970, 0, 1); // Epoch
+    t.setSeconds(seconds);
+    return t;
+}
+
 export function VideoCard({video}) {
     let video_url = `/videos/video/${video.id}`;
     let upload_date = uploadDate(video.upload_date);

app/src/components/Files.js

@@ -0,0 +1,119 @@
+import React from "react";
+import FileBrowser from 'react-keyed-file-browser';
+import {Button, Confirm, Header, Icon, Modal} from "semantic-ui-react";
+import 'react-keyed-file-browser/dist/react-keyed-file-browser.css';
+import {deleteFile, getFiles} from "../api";
+import {humanFileSize, secondsToDateTime} from "./Common";
+
+const icons = {
+    File: <Icon name='file'/>,
+    Folder: <Icon name='folder'/>,
+    FolderOpen: <Icon name='folder open'/>,
+    Image: <Icon name='file image'/>,
+    PDF: <Icon name='file pdf'/>,
+};
+
+class Files extends React.Component {
+    constructor(props) {
+        super(props);
+        this.state = {
+            deleteOpen: false,
+            files: [],
+            openFolders: [],
+            previewOpen: false,
+            selectedFile: null,
+        };
+    }
+
+    async componentDidMount() {
+        await this.fetchFiles();
+    }
+
+    fetchFiles = async () => {
+        let {files} = await getFiles(this.state.openFolders);
+        for (let i = 0; i < files.length; i++) {
+            files[i]['modified'] = secondsToDateTime(files[i]['modified']);
+        }
+        this.setState({files: files});
+    }
+
+    handleFolderChange = async (file, browserProps) => {
+        let openFolders = Object.keys(browserProps.openFolders);
+        this.setState({openFolders}, this.fetchFiles);
+    }
+
+    onDownloadFile = async () => {
+        // Open file in new tab/window.
+        window.open(`/media/${this.state.selectedFile.key}`);
+    }
+
+    onSelectFile = async (file) => {
+        this.setState({selectedFile: file}, this.openPreview);
+    }
+
+    onDeleteFile = async () => {
+        await deleteFile(this.state.selectedFile.key);
+        await this.fetchFiles();
+        this.closePreview();
+    }
+
+    openPreview = () => {
+        this.setState({previewOpen: true});
+    }
+
+    closePreview = () => {
+        this.setState({previewOpen: false, selectedFile: null, deleteOpen: false});
+    }
+
+    openDelete = () => {
+        this.setState({deleteOpen: true});
+    }
+
+    closeDelete = () => {
+        this.setState({deleteOpen: false});
+    }
+
+    render() {
+        return <>
+            {this.state.selectedFile &&
+                <Modal closeIcon
+                       open={this.state.previewOpen}
+                       onClose={this.closePreview}
+                >
+                    <Modal.Header><Icon name='file'/>File Preview</Modal.Header>
+                    <Modal.Content>
+                        <Header>{this.state.selectedFile.name}</Header>
+                        <p>{humanFileSize(this.state.selectedFile.size)}</p>
+                    </Modal.Content>
+                    <Modal.Actions>
+                        <Button color='red' onClick={this.openDelete} floated='left'>Delete</Button>
+                        <Confirm
+                            open={this.state.deleteOpen}
+                            content={`Are you sure you want to delete ${this.state.selectedFile.key}`}
+                            onCancel={this.closeDelete}
+                            onConfirm={this.onDeleteFile}
+                        />
+                        <Button color='green' onClick={this.onDownloadFile}>Download</Button>
+                    </Modal.Actions>
+                </Modal>}
+            <FileBrowser
+                showActionBar={false}
+                canFilter={false}
+                files={this.state.files}
+                icons={icons}
+                onFolderOpen={this.handleFolderChange}
+                onFolderClose={this.handleFolderChange}
+                onSelectFile={this.onSelectFile}
+                detailRenderer={() => <></>} // Hide the preview that the 3rd party provided.
+            />
+        </>
+    }
+}
+
+export class FilesRoute extends React.Component {
+    render() {
+        return (
+            <Files/>
+        );
+    }
+}

app/src/components/Nav.js

@@ -15,6 +15,7 @@ const links = [
     },
     {text: 'Inventory', to: '/inventory', exact: true},
     {text: 'Archive', to: '/archive', exact: true},
+    {text: 'Files', to: '/files', exact: true},
     {
         text: 'Apps', links: [
             {to: '/apps/otp', text: 'One Time Pad', exact: true},

docker/app/Dockerfile

@@ -8,7 +8,7 @@ ENV PATH /app/node_modules/.bin:$PATH
 COPY app/public /app/public
 COPY app/src /app/src
 COPY app/package.json /app/
-RUN rm -rf /app/node_modules
+RUN rm -rf /app/node_modules /app/package-lock.json
 # Increase network timeout because a Raspberry Pi is slow.
 RUN yarn --network-timeout 100000
 

modules/archive/lib.py

@@ -15,7 +15,7 @@ from sqlalchemy.orm import Session
 
 from modules.archive.models import Domain, Archive
 from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir
-from wrolpi.dates import now, strptime_ms, strftime_ms
+from wrolpi.dates import now, strptime_ms, strftime_ms, Seconds
 from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
 from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive
 from wrolpi.vars import DOCKERIZED, PYTEST
@@ -69,7 +69,7 @@ def get_new_archive_files(url: str):
     return ret
 
 
-ARCHIVE_TIMEOUT = 10 * 60  # Wait at most 10 minutes for response.
+ARCHIVE_TIMEOUT = Seconds.minute * 10  # Wait at most 10 minutes for response.
 
 
 def request_archive(url: str):

modules/files/__init__.py

@@ -0,0 +1 @@
+from .api import bp

modules/files/api.py

@@ -0,0 +1,66 @@
+import pathlib
+from typing import List
+from urllib.request import Request
+
+from sanic import response
+
+from wrolpi.common import get_media_directory
+from wrolpi.errors import InvalidFile
+from wrolpi.root_api import get_blueprint, json_response
+from wrolpi.schema import validate_doc
+from . import lib, schema
+
+bp = get_blueprint('Files', '/api/files')
+
+
+def paths_to_files(paths: List[pathlib.Path]):
+    media_directory = get_media_directory()
+    new_files = []
+    for path in paths:
+        stat = path.stat()
+        key = path.relative_to(media_directory)
+        modified = stat.st_mtime
+        if path.is_dir():
+            key = f'{key}/'
+            new_files.append(dict(
+                key=key,
+                modified=modified,
+                url=key,
+                name=path.name,
+            ))
+        else:
+            # A File should know it's size.
+            new_files.append(dict(
+                key=key,
+                modified=modified,
+                size=stat.st_size,
+                url=key,
+                name=path.name,
+            ))
+    return new_files
+
+
+@bp.post('/')
+@validate_doc(
+    'List files in a directory',
+    consumes=schema.FilesRequest,
+)
+async def get_files(_: Request, data: dict):
+    directories = data.get('directories') or []
+
+    files = lib.list_files(directories)
+    files = paths_to_files(files)
+    return json_response({'files': files})
+
+
+@bp.post('/delete')
+@validate_doc(
+    'Delete a single file.  Returns an error if WROL Mode is enabled.',
+    consumes=schema.DeleteRequest,
+)
+async def delete_file(_: Request, data: dict):
+    file = data.get('file')
+    if not file:
+        raise InvalidFile('file cannot be empty')
+    lib.delete_file(file)
+    return response.empty()

modules/files/conftest.py

@@ -0,0 +1,21 @@
+import pathlib
+from typing import List
+
+import pytest
+
+
+@pytest.fixture
+def make_files_structure(test_directory):
+    def create_files(paths: List[str]) -> List[pathlib.Path]:
+        files = []
+        for name in paths:
+            path = test_directory / name
+            if name.endswith('/'):
+                path.mkdir()
+            else:
+                path.parent.mkdir(parents=True, exist_ok=True)
+                path.touch()
+            files.append(path)
+        return files
+
+    return create_files

modules/files/lib.py

@@ -0,0 +1,60 @@
+from pathlib import Path
+from typing import List
+
+from wrolpi.common import get_media_directory, wrol_mode_check
+from wrolpi.errors import InvalidFile
+
+
+def filter_parent_directories(directories: List[Path]) -> List[Path]:
+    """
+    Remove parent directories if their children are in the list.
+
+    >>> filter_parent_directories([Path('foo'), Path('foo/bar'), Path('baz')])
+    [Path('foo/bar'), Path('baz')]
+    """
+    unique_children = set()
+    for directory in sorted(directories):
+        for parent in directory.parents:
+            # Remove any parent of this child.
+            if parent in unique_children:
+                unique_children.remove(parent)
+        unique_children.add(directory)
+
+    # Restore the original order.
+    new_directories = [i for i in directories if i in unique_children]
+    return new_directories
+
+
+def list_files(directories: List[str]) -> List[Path]:
+    """
+    List all files down to the directories provided.  This includes all parent directories of the directories.
+    """
+    media_directory = get_media_directory()
+
+    # Always display the media_directory files.
+    paths = list(media_directory.iterdir())
+
+    if directories:
+        directories = [media_directory / i for i in directories if i]
+        directories = filter_parent_directories(directories)
+        for directory in directories:
+            directory = media_directory / directory
+            for parent in directory.parents:
+                is_relative_to = str(media_directory).startswith(str(parent))
+                if parent == Path('.') or is_relative_to:
+                    continue
+                paths.extend(parent.iterdir())
+            paths.extend(directory.iterdir())
+
+    return paths
+
+
+@wrol_mode_check
+def delete_file(file: str):
+    """
+    Delete a file in the media directory.
+    """
+    file = get_media_directory() / file
+    if file.is_dir() or not file.is_file():
+        raise InvalidFile(f'Invalid file {file}')
+    file.unlink()

modules/files/schema.py

@@ -0,0 +1,9 @@
+from sanic_openapi import doc
+
+
+class FilesRequest:
+    directories = doc.List(doc.String())
+
+
+class DeleteRequest:
+    file = doc.String()

modules/files/test/test_api.py

@@ -0,0 +1,116 @@
+import json
+from http import HTTPStatus
+from itertools import zip_longest
+from unittest import mock
+
+import pytest
+
+from wrolpi.errors import API_ERRORS, WROLModeEnabled
+
+
+def test_list_files_api(test_client, make_files_structure, test_directory):
+    files = [
+        'archives/bar.txt',
+        'archives/baz/bar.txt',
+        'archives/baz/foo.txt',
+        'archives/foo.txt',
+        'empty directory/',
+        'videos/other video.mp4',
+        'videos/some video.mp4',
+    ]
+    files = make_files_structure(files)
+    files[0].write_text('bar contents')
+
+    def check_get_files(directories, expected_files):
+        request, response = test_client.post('/api/files', content=json.dumps({'directories': directories}))
+        assert not response.json.get('errors')
+        results = sorted(response.json['files'], key=lambda i: i['key'])
+        for f1, f2 in zip_longest(results, expected_files):
+            if f1 is None or f2 is None:
+                assert f1 == f2
+            for key in f2.keys():
+                assert f1[key] == f2[key], f'{f1} != {f2}'
+
+    # Requesting no directories results in the top-level results.
+    expected = [dict(key='archives/'), dict(key='empty directory/'), dict(key='videos/')]
+    check_get_files([], expected)
+    # empty directory is empty
+    check_get_files(['empty directory'], expected)
+
+    expected = [
+        dict(key='archives/'),
+        dict(key='archives/bar.txt', size=12),
+        dict(key='archives/baz/'),
+        dict(key='archives/foo.txt'),
+        dict(key='empty directory/'),
+        dict(key='videos/'),
+    ]
+    check_get_files(['archives'], expected)
+
+    # Sub-directories are supported.
+    expected = [
+        dict(key='archives/'),
+        dict(key='archives/bar.txt', size=12),
+        dict(key='archives/baz/'),
+        dict(key='archives/baz/bar.txt'),
+        dict(key='archives/baz/foo.txt'),
+        dict(key='archives/foo.txt'),
+        dict(key='empty directory/'),
+        dict(key='videos/'),
+    ]
+    check_get_files(['archives/baz'], expected)
+
+    expected = [
+        dict(key='archives/'),
+        dict(key='archives/bar.txt', size=12),
+        dict(key='archives/baz/'),
+        dict(key='archives/foo.txt'),
+        dict(key='empty directory/'),
+        dict(key='videos/'),
+        dict(key='videos/other video.mp4'),
+        dict(key='videos/some video.mp4'),
+    ]
+    check_get_files(['archives', 'videos'], expected)
+    # Order does not matter.
+    check_get_files(['archives', 'videos', 'empty directory'], expected)
+    check_get_files(['archives', 'empty directory', 'videos'], expected)
+
+
+def test_delete_file(test_client, make_files_structure, test_directory):
+    files = ['bar.txt', 'baz/']
+    make_files_structure(files)
+
+    request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'bar.txt'}))
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    assert not (test_directory / 'bar.txt').is_file()
+    assert (test_directory / 'baz').is_dir()
+
+    request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'baz'}))
+    assert response.status_code == HTTPStatus.BAD_REQUEST
+    assert not (test_directory / 'bar.txt').is_file()
+    assert (test_directory / 'baz').is_dir()
+
+    request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'bad file'}))
+    assert response.status_code == HTTPStatus.BAD_REQUEST
+
+
+@pytest.mark.parametrize(
+    'file', [
+        '',
+    ]
+)
+def test_delete_invalid_file(test_client, file):
+    with mock.patch('modules.files.api.lib.delete_file') as mock_delete_file:
+        request, response = test_client.post('/api/files/delete', content=json.dumps({'file': file}))
+        assert response.status_code == HTTPStatus.BAD_REQUEST
+        mock_delete_file.assert_not_called()
+
+
+def test_delete_wrol_mode(test_client):
+    """
+    Can't delete a file when WROL Mode is enabled.
+    """
+    with mock.patch('wrolpi.common.wrol_mode_enabled', lambda: True):
+        request, response = test_client.post('/api/files/delete', content=json.dumps({'file': 'foo'}))
+        assert response.status_code == HTTPStatus.FORBIDDEN
+        assert response.json['code'] == API_ERRORS[WROLModeEnabled]['code']

modules/files/test/test_lib.py

@@ -0,0 +1,124 @@
+from pathlib import Path
+
+import pytest
+
+from modules.files import lib
+from wrolpi.errors import InvalidFile
+
+
+def test_list_files(make_files_structure, test_directory):
+    files = [
+        'archives/bar.txt',
+        'archives/baz/bar.txt',
+        'archives/baz/foo.txt',
+        'archives/foo.txt',
+        'empty directory/',
+        'videos/other video.mp4',
+        'videos/some video.mp4',
+    ]
+    make_files_structure(files)
+
+    def check_list_files(path, expected):
+        result = lib.list_files(path)
+        result = sorted(result)
+        assert [str(i.relative_to(test_directory)) for i in result] == expected
+
+    check_list_files([], ['archives', 'empty directory', 'videos'])
+    # Falsey directories are ignored.
+    check_list_files([None, ''], ['archives', 'empty directory', 'videos'])
+    check_list_files(['empty directory'], ['archives', 'empty directory', 'videos'])
+    check_list_files(['empty directory'], ['archives', 'empty directory', 'videos'])
+    # Trailing slash is ignored.
+    check_list_files(['empty directory/'], ['archives', 'empty directory', 'videos'])
+
+    check_list_files(['archives'], [
+        'archives',
+        'archives/bar.txt',
+        'archives/baz',
+        'archives/foo.txt',
+        'empty directory',
+        'videos',
+    ])
+    check_list_files(['archives/baz'], [
+        'archives',
+        'archives/bar.txt',
+        'archives/baz',
+        'archives/baz/bar.txt',
+        'archives/baz/foo.txt',
+        'archives/foo.txt',
+        'empty directory',
+        'videos',
+    ])
+    # Including duplicate directories does not change results.
+    check_list_files(['archives/baz', 'archives'], [
+        'archives',
+        'archives/bar.txt',
+        'archives/baz',
+        'archives/baz/bar.txt',
+        'archives/baz/foo.txt',
+        'archives/foo.txt',
+        'empty directory',
+        'videos',
+    ])
+    check_list_files(['videos'], [
+        'archives',
+        'empty directory',
+        'videos',
+        'videos/other video.mp4',
+        'videos/some video.mp4',
+    ])
+    # All files and directories are listed.
+    check_list_files(['videos', 'archives/baz'], [
+        'archives',
+        'archives/bar.txt',
+        'archives/baz',
+        'archives/baz/bar.txt',
+        'archives/baz/foo.txt',
+        'archives/foo.txt',
+        'empty directory',
+        'videos',
+        'videos/other video.mp4',
+        'videos/some video.mp4',
+    ])
+
+
+@pytest.mark.parametrize(
+    'directories,expected', [
+        ([Path('foo')], [Path('foo')]),
+        ([Path('foo'), Path('bar')], [Path('foo'), Path('bar')]),
+        ([Path('foo'), Path('foo/bar'), Path('baz')], [Path('foo/bar'), Path('baz')]),
+        ([Path('foo/bar'), Path('foo'), Path('baz')], [Path('foo/bar'), Path('baz')]),
+        ([Path('baz'), Path('foo/bar'), Path('foo')], [Path('baz'), Path('foo/bar')]),
+        ([Path('foo/bar'), Path('foo'), Path('foo/bar/baz')], [Path('foo/bar/baz')]),
+    ]
+)
+def test_filter_parent_directories(directories, expected):
+    assert lib.filter_parent_directories(directories) == expected
+
+
+def test_delete_file(make_files_structure, test_directory):
+    """
+    File in the media directory can be deleted.
+    """
+    files = [
+        'archives/foo.txt',
+        'bar.txt',
+        'baz/',
+    ]
+    make_files_structure(files)
+
+    lib.delete_file('bar.txt')
+    assert (test_directory / 'archives/foo.txt').is_file()
+    assert not (test_directory / 'bar.txt').is_file()
+    assert (test_directory / 'baz').is_dir()
+
+    lib.delete_file('archives/foo.txt')
+    assert not (test_directory / 'archives/foo.txt').is_file()
+    assert not (test_directory / 'bar.txt').is_file()
+
+    with pytest.raises(InvalidFile):
+        lib.delete_file('baz')
+    with pytest.raises(InvalidFile):
+        lib.delete_file('does not exist')
+
+    assert (test_directory / 'baz').is_dir()

modules/videos/lib.py

@@ -10,7 +10,7 @@ from sqlalchemy.orm import Session
 
 from wrolpi.common import logger, chunks, get_config
 from wrolpi.common import save_settings_config
-from wrolpi.dates import from_timestamp
+from wrolpi.dates import from_timestamp, Seconds
 from wrolpi.db import get_db_curs, get_db_session, optional_session
 from wrolpi.media_path import MediaPath
 from wrolpi.vars import PYTEST
@@ -21,7 +21,7 @@ from .models import Channel, Video
 
 logger = logger.getChild(__name__)
 
-DEFAULT_DOWNLOAD_FREQUENCY = 60 * 60 * 24 * 7  # weekly
+DEFAULT_DOWNLOAD_FREQUENCY = Seconds.week
 
 
 def refresh_channel_videos(channel: Channel):

wrolpi/conftest.py

@@ -15,7 +15,7 @@ from sqlalchemy.engine import Engine, create_engine
 from sqlalchemy.orm import Session, sessionmaker
 
 from wrolpi.common import set_test_media_directory, Base, get_example_config
-from wrolpi.dates import fake_now  # noqa
+from wrolpi.dates import set_test_now
 from wrolpi.db import postgres_engine, get_db_args
 from wrolpi.downloader import DownloadManager
 from wrolpi.root_api import BLUEPRINTS, api_app
@@ -122,3 +122,11 @@ def test_client() -> SanicTestClient:
 def test_download_manager():
     manager = DownloadManager()
     return manager
+
+
+@pytest.fixture
+def fake_now():
+    try:
+        yield set_test_now
+    finally:
+        set_test_now(None)  # reset now() to its original functionality.

wrolpi/dates.py

@@ -2,7 +2,6 @@ import logging
 from datetime import datetime, date, timezone
 from enum import Enum
 
-import pytest
 import pytz
 from sqlalchemy import types
 
@@ -20,17 +19,14 @@ class Seconds(int, Enum):
     hour = minute * 60
     day = hour * 24
     week = day * 7
+    year = day * 366
 
 
-@pytest.fixture
-def fake_now():
-    def _(dt: datetime):
-        if not dt.tzinfo:
-            dt = local_timezone(dt)
-        global TEST_DATETIME
-        TEST_DATETIME = dt
-
-    return _
+def set_test_now(dt: datetime):
+    if dt and not dt.tzinfo:
+        dt = local_timezone(dt)
+    global TEST_DATETIME
+    TEST_DATETIME = dt
 
 
 def set_timezone(tz: pytz.timezone):

wrolpi/errors.py

@@ -136,6 +136,10 @@ class UnrecoverableDownloadError(APIError):
     pass
 
 
+class InvalidFile(APIError):
+    pass
+
+
 error_codes = iter(range(1, 1000))
 
 API_ERRORS = {
@@ -299,4 +303,9 @@ API_ERRORS = {
         'message': 'Download experienced an error which cannot be recovered.  Download will be deleted.',
         'status': HTTPStatus.INTERNAL_SERVER_ERROR,
     },
+    InvalidFile: {
+        'code': next(error_codes),
+        'message': 'File does not exist or is a directory',
+        'status': HTTPStatus.BAD_REQUEST,
+    }
 }