Zim/Kiwix (Wikipedia) feature

* A new zim module has been added, this module allows several things to interact with Zim files.
  * Allow searching of Zim files using libzim.
  * Allow downloading of Kiwix projects (wikipedia, wikibooks, etc) on a schedule which replaces the outdated files.
  * Creation of a new zim docker container which allows viewing of the Zim file contents.

* This commit also adds move/rename/mkdir features in the File Browser.
  * File tags are preserved when moving/renaming.

.run/Python WROLPi & Modules Tests.run.xml

@@ -1,17 +1,17 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Python tests in wrolpi" type="tests" factoryName="Autodetect" nameIsGenerated="true">
+  <configuration default="false" name="Python WROLPi &amp; Modules Tests" type="tests" factoryName="Autodetect">
     <module name="wrolpi" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />
     <option name="SDK_HOME" value="" />
-    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/wrolpi" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
     <option name="IS_MODULE_SDK" value="true" />
     <option name="ADD_CONTENT_ROOTS" value="true" />
     <option name="ADD_SOURCE_ROOTS" value="true" />
     <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
-    <option name="_new_additionalArguments" value="&quot;&quot;" />
-    <option name="_new_target" value="&quot;$PROJECT_DIR$/wrolpi&quot;" />
-    <option name="_new_targetType" value="&quot;PATH&quot;" />
+    <option name="_new_additionalArguments" value="&quot;wrolpi modules&quot;" />
+    <option name="_new_target" value="&quot;&quot;" />
+    <option name="_new_targetType" value="&quot;CUSTOM&quot;" />
     <method v="2" />
   </configuration>
 </component>

50x.html

@@ -37,12 +37,23 @@
 <h2>Can't start the services?</h2>
 <p>You can view the media directory by <a id="media_link" href="/media">navigating to /media</a></p>
 
+<p>The API may be available <a id="api_link" href="#">by navigating to port 8081</a></p>
+
 <p>The map may be available <a id="map_link" href="#">by navigating to port 8084</a></p>
 
+<p>Kiwix may be available <a id="kiwix_link" href="#">by navigating to port 8085</a></p>
+
 <script>
-    // Get the current hostname (probably an IP) link to my :8084 above.
+    // Get the current hostname (probably an IP) link to my other ports above.
     let hostname = window.location.hostname;
 
+    let api_a = document.getElementById('map_link');
+    if (api_a) {
+        let api_url = `http://${hostname}:8081`;
+        api_a.setAttribute('href', api_url);
+        api_a.textContent = `by navigating to ${api_url}`;
+    }
+
     let media_a = document.getElementById('media_link');
     if (media_a) {
         let media_url = `http://${hostname}/media`;
@@ -56,6 +67,13 @@
         map_a.setAttribute('href', map_url);
         map_a.textContent = `by navigating to ${map_url}`;
     }
+
+    let kiwix_a = document.getElementById('map_link');
+    if (kiwix_a) {
+        let kiwix_url = `http://${hostname}:8085`;
+        kiwix_a.setAttribute('href', kiwix_url);
+        kiwix_a.textContent = `by navigating to ${kiwix_url}`;
+    }
 </script>
 
 </body>

README.md

@@ -122,6 +122,12 @@ Install WROLPi onto a fresh Raspberry Pi.
 
 # Charter
 
+## Guiding Principals
+
+1. Storage is cheaper than power.
+2. Two is one, one is none.
+3. In WROL mode, run silently.
+
 ## Physical properties
 
 1. A WROLPi instance should be capable of running with a minimal of hardware:
@@ -129,7 +135,7 @@ Install WROLPi onto a fresh Raspberry Pi.
     * SD card
     * External USB hard drive
     * Power supply and cables
-    * WiFi USER device such as a phone, tablet, or laptop.
+    * Wi-Fi USER device such as a phone, tablet, or laptop.
 2. A WROLPi instance should consume a minimal amount of electricity during WROL event. It is expected power will be
    scarce when WROLPi is needed most.
 

alembic/versions/3b6918aeca4b_file_group.py

@@ -76,9 +76,12 @@ def upgrade():
         ) STORED
     )''')
 
-    session.execute('ALTER TABLE video ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
-    session.execute('ALTER TABLE ebook ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
-    session.execute('ALTER TABLE archive ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
+    session.execute(
+        'ALTER TABLE video ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
+    session.execute(
+        'ALTER TABLE ebook ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
+    session.execute(
+        'ALTER TABLE archive ADD COLUMN file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL UNIQUE')
 
     session.execute('''
     CREATE TABLE tag (

alembic/versions/751aecf46b88_zim.py

@@ -0,0 +1,67 @@
+"""Zim
+
+Revision ID: 751aecf46b88
+Revises: 75d34a2b442d
+Create Date: 2023-06-04 19:48:41.305111
+
+"""
+import os
+
+from alembic import op
+from sqlalchemy.orm import Session
+
+# revision identifiers, used by Alembic.
+revision = '751aecf46b88'
+down_revision = '75d34a2b442d'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    session.execute('''
+    CREATE TABLE zim (
+        id SERIAL PRIMARY KEY,
+        path TEXT NOT NULL UNIQUE,
+        file_group_id BIGINT REFERENCES file_group(id) ON DELETE CASCADE NOT NULL
+    )''')
+
+    session.execute('''
+    CREATE TABLE tag_zim (
+        tag_id INTEGER REFERENCES tag(id),
+        zim_id INTEGER REFERENCES zim(id) ON DELETE CASCADE,
+        zim_entry TEXT NOT NULL,
+        created_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP,
+        UNIQUE (tag_id, zim_id, zim_entry)
+    )''')
+
+    session.execute('''
+    CREATE TABLE zim_subscription (
+        id SERIAL PRIMARY KEY,
+        name TEXT NOT NULL UNIQUE,
+        language TEXT NOT NULL,
+        download_id INTEGER NOT NULL REFERENCES download(id) ON DELETE CASCADE
+    )
+    ''')
+
+    # We want to index all Zim files using `zim_modeler`
+    session.execute("UPDATE file_group SET indexed=false WHERE primary_path ILIKE '%.zim'")
+
+    if not DOCKERIZED:
+        session.execute('ALTER TABLE public.zim OWNER TO wrolpi')
+        session.execute('ALTER TABLE public.tag_zim OWNER TO wrolpi')
+        session.execute('ALTER TABLE public.zim_subscription OWNER TO wrolpi')
+
+
+def downgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    session.execute('DROP TABLE IF EXISTS zim_subscription')
+    session.execute('DROP TABLE IF EXISTS tag_zim')
+    session.execute('DROP TABLE IF EXISTS zim')
+    session.execute("UPDATE file_group SET indexed=false WHERE primary_path ILIKE '%.zim'")

alembic/versions/84b6993a6091_drop_channel_next_download.py

@@ -6,10 +6,10 @@ Create Date: 2021-12-27 16:42:32.099796
 
 """
 import os
+
 from alembic import op
 from sqlalchemy.orm import Session
 
-
 # revision identifiers, used by Alembic.
 revision = '84b6993a6091'
 down_revision = '8df411232c17'

alembic/versions/aac864072193_increase_file_size_column.py

@@ -6,10 +6,10 @@ Create Date: 2022-06-22 17:34:33.635528
 
 """
 import os
+
 from alembic import op
 from sqlalchemy.orm import Session
 
-
 # revision identifiers, used by Alembic.
 revision = 'aac864072193'
 down_revision = '467235201c70'

alembic/versions/b97c1313f986_video_modification_datetime.py

@@ -6,10 +6,10 @@ Create Date: 2022-01-25 08:00:26.944183
 
 """
 import os
+
 from alembic import op
 from sqlalchemy.orm import Session
 
-
 # revision identifiers, used by Alembic.
 revision = 'b97c1313f986'
 down_revision = '3c33b1dae069'

alembic/versions/bf704baa185d_zim_flag.py

@@ -0,0 +1,33 @@
+"""Zim flag
+
+Revision ID: bf704baa185d
+Revises: 751aecf46b88
+Create Date: 2023-07-01 20:00:35.601752
+
+"""
+import os
+
+from alembic import op
+from sqlalchemy.orm import Session
+
+# revision identifiers, used by Alembic.
+revision = 'bf704baa185d'
+down_revision = '751aecf46b88'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    session.execute('ALTER TABLE wrolpi_flag ADD COLUMN outdated_zims BOOLEAN DEFAULT FALSE')
+
+
+def downgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    session.execute('ALTER TABLE wrolpi_flag DROP COLUMN outdated_zims')

alembic/versions/d2eace02f5b9_file_path_on_cascade.py

@@ -6,10 +6,10 @@ Create Date: 2022-09-28 15:14:43.306819
 
 """
 import os
+
 from alembic import op
 from sqlalchemy.orm import Session
 
-
 # revision identifiers, used by Alembic.
 revision = 'd2eace02f5b9'
 down_revision = '70db5ba8763d'

alembic/versions/f2c6326cad29_video_source_id_index.py

@@ -6,10 +6,10 @@ Create Date: 2022-01-10 08:27:39.082067
 
 """
 import os
+
 from alembic import op
 from sqlalchemy.orm import Session
 
-
 # revision identifiers, used by Alembic.
 revision = 'f2c6326cad29'
 down_revision = 'f8c65e7e6802'

alembic/versions/f8c65e7e6802_channel_refresh.py

@@ -6,10 +6,10 @@ Create Date: 2022-01-07 14:28:21.787842
 
 """
 import os
+
 from alembic import op
 from sqlalchemy.orm import Session
 
-
 # revision identifiers, used by Alembic.
 revision = 'f8c65e7e6802'
 down_revision = '62acddc55091'

alembic/versions/f9928a3b6bc5_file_directories.py

@@ -6,6 +6,7 @@ Create Date: 2023-04-18 16:58:40.043115
 
 """
 import os
+
 from alembic import op
 from sqlalchemy.orm import Session
 

app/public/epub/epub.html

@@ -30,8 +30,8 @@ either expressed or implied, of the FreeBSD Project. -->
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
-    <script src="/jszip.min.js"></script>
-    <script src="/epub.js"></script>
+    <script src="/epub/jszip.min.js"></script>
+    <script src="/epub/epub.js"></script>
 </head>
 <body>
 

app/public/index.html

@@ -2,14 +2,17 @@
 <html lang="en" style="height: 100%">
 <head>
     <meta charset="utf-8"/>
-    <link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
     <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
     <meta name="theme-color" content="#000000"/>
     <meta
             name="description"
             content="The WROLPI webapp"
     />
-    <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png"/>
+    <link rel="icon" href="%PUBLIC_URL%/favicon.ico"/>
+    <link rel="apple-touch-icon" sizes="180x180" href="%PUBLIC_URL%/apple-touch-icon.png">
+    <link rel="icon" type="image/png" sizes="32x32" href="%PUBLIC_URL%/favicon-32x32.png">
+    <link rel="icon" type="image/png" sizes="16x16" href="%PUBLIC_URL%/favicon-16x16.png">
+    <link rel="mask-icon" href="%PUBLIC_URL%/safari-pinned-tab.svg" color="#5bbad5">
     <!--
       manifest.json provides metadata used when your web app is installed on a
       user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/

app/public/manifest.json

@@ -3,19 +3,14 @@
   "name": "WROLPi App",
   "icons": [
     {
-      "src": "favicon.ico",
-      "sizes": "64x64 32x32 24x24 16x16",
-      "type": "image/x-icon"
+      "src": "/android-chrome-192x192.png",
+      "sizes": "192x192",
+      "type": "image/png"
     },
     {
-      "src": "logo192.png",
-      "type": "image/png",
-      "sizes": "191x191"
-    },
-    {
-      "src": "logo512.png",
-      "type": "image/png",
-      "sizes": "512x509"
+      "src": "/android-chrome-512x512.png",
+      "sizes": "512x512",
+      "type": "image/png"
     }
   ],
   "start_url": ".",

app/src/App.js

@@ -20,6 +20,7 @@ import {useEventsInterval} from "./Events";
 import {SemanticToastContainer} from "react-semantic-toasts-2";
 import {FilePreviewProvider} from "./components/FilePreview";
 import {TagsProvider} from "./Tags";
+import {ZimRoute} from "./components/Zim";
 
 function PageNotFound() {
     const {t} = useContext(ThemeContext);
@@ -63,6 +64,7 @@ const router = createBrowserRouter(createRoutesFromElements(<Route
     errorElement={<PageNotFound/>}
 >
     <Route index element={<Dashboard/>}/>
+    <Route path='search/*' element={<Dashboard/>}/>
     <Route path='donate' element={<Donate/>}/>
     <Route path='videos/video/:videoId' exact element={<VideoWrapper/>}/>
     <Route path='videos/channel/:channelId/video/:videoId' exact element={<VideoWrapper/>}/>
@@ -72,6 +74,7 @@ const router = createBrowserRouter(createRoutesFromElements(<Route
     <Route path="inventory/*" element={<InventoryRoute/>}/>
     <Route path='archive/*' element={<ArchiveRoute/>}/>
     <Route path='map/*' element={<MapRoute/>}/>
+    <Route path='zim/*' element={<ZimRoute/>}/>
     <Route path='files/*' element={<FilesRoute/>}/>
 </Route>));
 

app/src/Dashboard.js

@@ -1,9 +1,7 @@
 import {LoadStatistic, PageContainer, SearchInput, useTitle} from "./components/Common";
-import {useQuery} from "./hooks/customHooks";
 import React, {useContext, useState} from "react";
 import {StatusContext} from "./contexts/contexts";
 import {DownloadMenu} from "./components/Download";
-import {FilesSearchView} from "./components/Files";
 import {
     Button,
     Divider,
@@ -15,7 +13,7 @@ import {
     Statistic,
     StatisticGroup
 } from "./components/Theme";
-import {Link} from "react-router-dom";
+import {Link, useNavigate} from "react-router-dom";
 import {BandwidthProgressCombined, CPUUsageProgress} from "./components/admin/Status";
 import {ProgressPlaceholder} from "./components/Placeholder";
 import {GridColumn, GridRow, Icon, Message} from "semantic-ui-react";
@@ -24,8 +22,13 @@ import {refreshFiles} from "./api";
 import _ from "lodash";
 import {TagsDashboard} from "./Tags";
 import {Upload} from "./components/Upload";
+import {SearchView, useSearch} from "./components/Search";
+import {KiwixRestartMessage, OutdatedZimsMessage} from "./components/Zim";
+import {useSettings} from "./hooks/customHooks";
 
 function FlagsMessages({flags}) {
+    const {settings, fetchSettings} = useSettings();
+
     if (!flags) {
         return <></>
     }
@@ -33,6 +36,7 @@ function FlagsMessages({flags}) {
     let refreshing;
     let refreshRequired;
     let dbDown;
+    let kiwixRestart;
 
     // Do not tell the maintainer to refresh the files if they are already refreshing.
     if (flags.indexOf('refreshing') >= 0) {
@@ -55,6 +59,10 @@ function FlagsMessages({flags}) {
         </Message>;
     }
 
+    if (flags.indexOf('kiwix_restart') >= 0) {
+        kiwixRestart = <KiwixRestartMessage/>;
+    }
+
     if (flags.indexOf('db_up') === -1) {
         dbDown = <Message icon error>
             <Icon name='exclamation'/>
@@ -68,20 +76,18 @@ function FlagsMessages({flags}) {
     return <>
         {refreshing}
         {dbDown || refreshRequired}
+        {settings && settings['ignore_outdated_zims'] === false && flags.indexOf('outdated_zims') >= 0 ?
+            <OutdatedZimsMessage onClick={fetchSettings}/> : null}
+        {kiwixRestart}
     </>
 }
 
 export function Dashboard() {
     useTitle('Dashboard');
 
-    const {searchParams, updateQuery} = useQuery();
-    const searchStr = searchParams.get('q');
-    const activeTags = searchParams.getAll('tag');
-    const setSearchStr = (value) => {
-        updateQuery({q: value, o: null});
-    }
-
+    const {searchStr, setSearchStr, activeTags} = useSearch();
     const {status} = useContext(StatusContext);
+    const navigate = useNavigate();
     const wrol_mode = status ? status['wrol_mode'] : null;
 
     // Getters are Downloads or Uploads.
@@ -141,7 +147,7 @@ export function Dashboard() {
     // Only show dashboard parts if not searching.
     let body;
     if (searchStr || (activeTags && activeTags.length > 0)) {
-        body = <FilesSearchView/>;
+        body = <SearchView/>;
     } else {
         body = <>
             {!wrol_mode && getter}
@@ -161,6 +167,7 @@ export function Dashboard() {
                                  size='large'
                                  placeholder='Search everywhere...'
                                  actionIcon='search'
+                                 onClear={() => navigate('/')}
                                  style={{marginBottom: '2em'}}
                     />
                 </Grid.Column>

app/src/Tags.js

@@ -105,7 +105,7 @@ export function useTags() {
     }
 
     const TagLabelLink = ({name, props}) => {
-        const to = `/?tag=${name}`;
+        const to = `/search?tag=${name}`;
         const style = {marginLeft: '0.3em', marginRight: '0.3em'};
         try {
             // We prefer to use Link to avoid reloading the page, check if React Router is available, so we can use it.
@@ -440,40 +440,6 @@ export const TagsDashboard = () => {
     </Segment>
 }
 
-export const TagsDropdown = ({value = [], onChange, ...props}) => {
-    const {tagNames} = React.useContext(TagsContext);
-    const [activeTags, setActiveTags] = React.useState(value);
-
-    const handleChange = (e, {value}) => {
-        if (e) {
-            e.preventDefault();
-        }
-        if (!value) {
-            setActiveTags([]);
-            onChange([]);
-        } else {
-            setActiveTags(value);
-            onChange(value);
-        }
-    }
-
-    let options = [{key: null, text: '', value: ''}];
-    if (tagNames && tagNames.length > 0) {
-        const tagOptions = tagNames.map(i => {
-            return {key: i, text: i, value: i}
-        })
-        options = [...options, ...tagOptions];
-    }
-
-    return <Dropdown multiple selection clearable fluid
-                     options={options}
-                     placeholder='Tags'
-                     onChange={handleChange}
-                     value={activeTags}
-                     {...props}
-    />
-}
-
 export const TagsProvider = (props) => {
     const tagsValue = useTagsInterval();
 

app/src/api.js

@@ -1,4 +1,4 @@
-import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, emptyToNull, OTP_API, VIDEOS_API} from "./components/Common";
+import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, emptyToNull, OTP_API, VIDEOS_API, ZIM_API} from "./components/Common";
 import {toast} from "react-semantic-toasts-2";
 
 function timeoutPromise(ms, promise) {
@@ -146,6 +146,15 @@ export async function getVideo(video_id) {
 }
 
 export async function deleteVideos(videoIds) {
+    if (!videoIds || videoIds.length === 0) {
+        toast({
+            type: 'error',
+            title: 'Empty request',
+            description: 'Unable to delete Videos because no IDs were passed.',
+            time: 5000,
+        });
+    }
+    console.debug(`Deleting Videos: ${videoIds}`);
     const i = videoIds.join(',');
     let response = await apiDelete(`${VIDEOS_API}/video/${i}`);
     if (response.status !== 204) {
@@ -342,6 +351,15 @@ export async function deleteItems(itemIds) {
 }
 
 export async function deleteArchives(archiveIds) {
+    if (!archiveIds || archiveIds.length === 0) {
+        toast({
+            type: 'error',
+            title: 'Empty request',
+            description: 'Unable to delete Archives because no IDs were passed.',
+            time: 5000,
+        });
+    }
+    console.debug(`Deleting Archives: ${archiveIds}`);
     let i = archiveIds.join(',');
     try {
         return await apiDelete(`${API_URI}/archive/${i}`);
@@ -521,8 +539,8 @@ export async function refreshFiles() {
     return await apiPost(`${API_URI}/files/refresh`);
 }
 
-export async function refreshDirectoryFiles(directory) {
-    let body = {paths: [directory]};
+export async function refreshDirectoryFiles(paths) {
+    let body = {paths: paths};
     try {
         return await apiPost(`${API_URI}/files/refresh`, body);
     } catch (e) {
@@ -536,6 +554,21 @@ export async function refreshDirectoryFiles(directory) {
     }
 }
 
+export async function makeDirectory(path) {
+    const body = {path: path};
+    try {
+        await apiPost(`${API_URI}/files/directory`, body);
+    } catch (e) {
+        console.error(e);
+        toast({
+            type: 'error',
+            title: 'Unable to create directory',
+            description: 'Failed to create directory.  See server logs.',
+            time: 5000,
+        });
+    }
+}
+
 export async function getFiles(directories) {
     console.debug(`getFiles ${directories}`);
     let body = {directories: directories || []};
@@ -544,9 +577,18 @@ export async function getFiles(directories) {
     return files;
 }
 
-export async function deleteFile(file) {
-    let body = {file: file};
-    await apiPost(`${API_URI}/files/delete`, body);
+export async function deleteFile(paths) {
+    let body = {paths: paths};
+    const response = await apiPost(`${API_URI}/files/delete`, body);
+    if (response.status === 409) {
+        const content = await response.json();
+        toast({
+            type: 'error',
+            title: 'Delete error',
+            description: content['message'],
+            time: 5000,
+        });
+    }
 }
 
 export async function fetchFilesProgress() {
@@ -702,7 +744,7 @@ export async function getTags() {
     }
 }
 
-export async function addTag(fileGroup, name) {
+export async function tagFileGroup(fileGroup, name) {
     const body = {tag_name: name};
 
     const {id, primary_path, path} = fileGroup;
@@ -721,7 +763,7 @@ export async function addTag(fileGroup, name) {
     }
 }
 
-export async function removeTag(fileGroup, name) {
+export async function untagFileGroup(fileGroup, name) {
     const body = {tag_name: name};
 
     const {id, primary_path, path} = fileGroup;
@@ -817,13 +859,118 @@ export async function searchDirectories(name) {
     }
 }
 
-export async function uploadFiles(files, destination) {
-    const data = new FormData();
-    for (let i = 0; i < files.length; i++) {
-        data.append('file', files[i]);
+export async function renamePath(path, newName) {
+    const body = {path, new_name: newName};
+    const response = await apiPost(`${API_URI}/files/rename`, body);
+    if (response.status !== 204) {
+        toast({type: 'error', title: 'Error', description: 'Failed to rename!', time: 5000});
     }
-    data.append('destination', destination);
-    console.log(data);
+}
+
+export async function movePaths(destination, paths) {
+    const body = {destination, paths};
+    const response = await apiPost(`${API_URI}/files/move`, body);
+    if (response.status !== 204) {
+        const content = await response.json();
+        toast({type: 'error', title: 'Error', description: content['api_error'], time: 5000});
+    }
+}
+
+export async function fetchZims() {
+    const response = await apiGet(`${API_URI}/zim`);
+    if (response.status === 200) {
+        const content = await response.json();
+        return {
+            zims: content['zims'],
+        }
+    } else {
+        toast({type: 'error', title: 'Error', description: 'Cannot fetch Zims', time: 5000});
+    }
+}
+
+export async function fetchZimSubscriptions() {
+    const response = await apiGet(`${API_URI}/zim/subscribe`);
+    if (response.status === 200) {
+        const content = await response.json();
+        return {
+            subscriptions: content['subscriptions'],
+            catalog: content['catalog'],
+            iso_639_codes: content['iso_639_codes'],
+        }
+    } else {
+        toast({type: 'error', title: 'Error', description: 'Cannot fetch Zim Subscriptions', time: 5000});
+    }
+}
+
+export async function searchZims(offset, limit, searchStr) {
+    offset = parseInt(offset || 0);
+    limit = parseInt(limit || DEFAULT_LIMIT);
+    let body = {offset, limit};
+    if (searchStr) {
+        body['search_str'] = searchStr;
+    }
+
+    console.debug('searching zims', body);
+    let response = await apiPost(`${ZIM_API}/search`, body);
+    if (response.status === 200) {
+        let data = await response.json();
+        return [data['zims'],];
+    } else {
+        toast({
+            type: 'error',
+            title: 'Unable to search zims',
+            description: 'Cannot search zims.  See server logs.',
+            time: 5000,
+        });
+        return [[], 0];
+    }
+}
+
+export async function searchZim(offset, limit, searchStr, zimId, activeTags) {
+    offset = parseInt(offset || 0);
+    limit = parseInt(limit || DEFAULT_LIMIT);
+    let body = {offset, limit, search_str: searchStr, tag_names: activeTags || []};
+
+    console.debug(`Searching Zim ${zimId} for: ${searchStr} tags: ${activeTags}`);
+    let response = await apiPost(`${ZIM_API}/search/${zimId}`, body);
+    if (response.status === 200) {
+        const content = await response.json();
+        const zim = content['zim'];
+        const searchLength = zim['search'].length;
+        console.debug(`Got ${searchLength} results for Zim ${zimId}`);
+        return content['zim'];
+    }
+}
+
+export async function tagZimEntry(zim_id, zim_entry, name) {
+    const body = {tag_name: name, zim_id: zim_id, zim_entry: zim_entry};
+
+    const uri = `${API_URI}/zim/tag`;
+    let response = await apiPost(uri, body);
+    if (response.status !== 201) {
+        console.error('Failed to add tag');
+    }
+}
+
+export async function untagZimEntry(zim_id, zim_entry, name) {
+    const body = {tag_name: name, zim_id: zim_id, zim_entry: zim_entry};
+
+    const uri = `${API_URI}/zim/untag`;
+    let response = await apiPost(uri, body);
+    if (response.status !== 201) {
+        console.error('Failed to add tag');
+    }
+}
+
+export async function zimSubscribe(name, language) {
+    const body = {name, language};
+    const response = await apiPost(`${API_URI}/zim/subscribe`, body);
+    return response.status === 201;
+}
+
+export async function zimUnsubscribe(id) {
+    const response = await apiDelete(`${API_URI}/zim/subscribe/${id}`);
+    return response.status === 204;
 }
 
 export async function fetchDecoded(vinNumber) {
@@ -837,3 +984,47 @@ export async function fetchDecoded(vinNumber) {
         return null;
     }
 }
+
+export async function searchEstimate(search_str, tagNames) {
+    const body = {search_str: search_str, tag_names: tagNames};
+    const response = await apiPost(`${API_URI}/search_estimate`, body);
+    if (response.status === 200) {
+        const content = await response.json();
+        const zims = content['zims'];
+        let zimSum = 0;
+        zims.map(i => zimSum += i['estimate']);
+        return {
+            file_groups: content['file_groups'],
+            zims: zims,
+            zimSum: zimSum,
+        }
+    }
+}
+
+export async function getOutdatedZims() {
+    const response = await apiGet(`${API_URI}/zim/outdated`);
+    if (response.status === 200) {
+        const content = await response.json();
+        return {
+            outdated: content['outdated'],
+            current: content['current'],
+        }
+    } else {
+        toast({
+            type: 'error',
+            title: 'Unable to fetch',
+            description: 'Cannot fetch outdated Zims.  See server logs.',
+            time: 5000,
+        });
+    }
+}
+
+export async function deleteOutdatedZims() {
+    const response = await apiDelete(`${API_URI}/zim/outdated`);
+    return response.status === 204;
+}
+
+export async function ignoreOutdatedZims() {
+    const response = await apiPost(`${API_URI}/zim/outdated`);
+    return response.status === 204;
+}

app/src/components/Archive.js

@@ -34,7 +34,7 @@ import {
     textEllipsis,
     useTitle
 } from "./Common";
-import {addTag, deleteArchives, postDownload, removeTag} from "../api";
+import {deleteArchives, postDownload, tagFileGroup, untagFileGroup} from "../api";
 import {Link, Route, Routes, useNavigate, useParams} from "react-router-dom";
 import Message from "semantic-ui-react/dist/commonjs/collections/Message";
 import {useArchive, useDomains, useSearchArchives, useSearchDomain} from "../hooks/customHooks";
@@ -146,12 +146,12 @@ function ArchivePage() {
     }
 
     const localAddTag = async (name) => {
-        await addTag(archiveFile, name);
+        await tagFileGroup(archiveFile, name);
         await fetchArchive();
     }
 
     const localRemoveTag = async (name) => {
-        await removeTag(archiveFile, name);
+        await untagFileGroup(archiveFile, name);
         await fetchArchive();
     }
 
@@ -359,7 +359,8 @@ function Archives() {
     const onDelete = async (e) => {
         e.preventDefault();
         setDeleteOpen(false);
-        const archiveIds = archives.filter(i => selectedArchives.indexOf(i['path']) >= 0).map(i => i['archive']['id']);
+        const archiveIds = archives.filter(i => selectedArchives.indexOf(i['primary_path']) >= 0)
+            .map(i => i['archive']['id']);
         await deleteArchives(archiveIds);
         await fetchArchives();
         setSelectedArchives([]);

app/src/components/Common.js

@@ -1,6 +1,6 @@
 import React, {useContext, useEffect, useState} from "react";
-import {Button as SButton, ButtonGroup, Card, Container, IconGroup, Input, Pagination, Search} from 'semantic-ui-react';
-import {Link, NavLink, useNavigate} from "react-router-dom";
+import {ButtonGroup, Card, Container, IconGroup, Input, Label, Pagination, Search} from 'semantic-ui-react';
+import {Link, NavLink, useNavigate, useSearchParams} from "react-router-dom";
 import Message from "semantic-ui-react/dist/commonjs/collections/Message";
 import {
     useDirectories,
@@ -23,7 +23,8 @@ import {
     Modal,
     ModalActions,
     ModalContent,
-    ModalDescription, ModalHeader,
+    ModalDescription,
+    ModalHeader,
     Popup,
     Statistic
 } from "./Theme";
@@ -36,6 +37,7 @@ export const API_URI = process.env && process.env.REACT_APP_API_URI ? process.en
 export const VIDEOS_API = `${API_URI}/videos`;
 export const ARCHIVES_API = `${API_URI}/archive`;
 export const OTP_API = `${API_URI}/otp`;
+export const ZIM_API = `${API_URI}/zim`;
 export const DEFAULT_LIMIT = 20;
 export const NAME = process.env && process.env.REACT_APP_NAME ? process.env.REACT_APP_NAME : null;
 
@@ -215,7 +217,7 @@ export const rssFrequencyOptions = [{key: 'once', text: 'Once', value: 0}, {
     key: 'daily', text: 'Daily', value: 86400
 }, {key: 'weekly', text: 'Weekly', value: 604800}, {key: 'biweekly', text: 'Biweekly', value: 1209600}, {
     key: '30days', text: '30 Days', value: 2592000
-}, {key: '90days', text: '90 Days', value: 7776000},];
+}, {key: '90days', text: '90 Days', value: 7776000}, {key: '180days', text: '180 Days', value: 15552000}];
 
 export function secondsToFrequency(seconds) {
     for (let i = 0; i < Object.keys(rssFrequencyOptions).length; i++) {
@@ -375,6 +377,7 @@ export function SearchInput({
                                 actionIcon,
                                 clearable,
                                 autoFocus = false,
+                                onClear = null,
                                 ...props
                             }) {
     let [value, setValue] = useState(searchStr || '');
@@ -384,6 +387,9 @@ export function SearchInput({
         // Clear the input when the "clear" button is clicked, search again.
         setValue('');
         onSubmit('');
+        if (onClear) {
+            onClear();
+        }
     }
 
     const localOnSubmit = (e) => {
@@ -437,9 +443,15 @@ export function textEllipsis(str, maxLength = 100, {side = "end", ellipsis = "..
 }
 
 export function TabLinks({links}) {
+    const [searchParams, setSearchParams] = useSearchParams();
+
+    const getTo = (to) => {
+        return `${to}?${searchParams.toString()}`;
+    }
+
     return <Menu tabular>
         {links.map((link) => <NavLink
-            to={link.to}
+            to={getTo(link.to)}
             className='item'
             style={{padding: '1em'}}
             key={link.to}
@@ -771,6 +783,12 @@ export function mimetypeIconName(mimetype, lowerPath = '') {
             } else if (lowerPath.endsWith('.azw3')) {
                 return 'book';
             }
+        } else if (mimetype.startsWith('application/vnd.openxmlformats-officedocument.spreadsheetml.') || mimetype.startsWith('application/vnd.ms-excel') || mimetype.startsWith('application/vnd.oasis.opendocument.spreadsheet')) {
+            return 'file excel'
+        } else if (mimetype.startsWith('application/vnd.openxmlformats-officedocument.wordprocessingml.') || mimetype.startsWith('application/msword') || mimetype.startsWith('application/vnd.oasis.opendocument.text')) {
+            return 'file word'
+        } else if (mimetype.startsWith('application/vnd.openxmlformats-officedocument.presentationml.') || mimetype.startsWith('application/vnd.ms-powerpoint') || mimetype.startsWith('application/vnd.oasis.opendocument.presentation')) {
+            return 'file powerpoint'
         }
     }
     if (lowerPath.endsWith('.pem')) {
@@ -1163,3 +1181,13 @@ export function SortButton({sorts = []}) {
         </ButtonGroup>
     </>
 }
+
+export function TagIcon() {
+    return <Label circular color='green' style={{padding: '0.5em', marginRight: '0.5em'}}>
+        <Icon name='tag' style={{margin: 0}}/>
+    </Label>
+}
+
+export function normalizeEstimate(estimate) {
+    return estimate > 999 ? '>999' : estimate;
+}

app/src/components/FileBrowser.js

@@ -1,11 +1,23 @@
-import {Button, Icon, Placeholder} from "./Theme";
-import {Checkbox, Confirm, PlaceholderLine, TableCell, TableFooter, TableHeaderCell, TableRow} from "semantic-ui-react";
-import {FileIcon, humanFileSize} from "./Common";
-import React, {useEffect} from "react";
-import {deleteFile} from "../api";
+import {Button, Header, Icon, Modal, ModalActions, ModalContent, ModalHeader, Placeholder, Table} from "./Theme";
+import {
+    Checkbox,
+    Confirm,
+    IconGroup,
+    Input,
+    PlaceholderLine,
+    TableBody,
+    TableCell,
+    TableFooter,
+    TableHeader,
+    TableHeaderCell,
+    TableRow
+} from "semantic-ui-react";
+import {DirectorySearch, FileIcon, humanFileSize} from "./Common";
+import React, {useEffect, useState} from "react";
+import {deleteFile, makeDirectory, movePaths, renamePath} from "../api";
 import _ from 'lodash';
 import {SortableTable} from "./SortableTable";
-import {useBrowseFiles} from "../hooks/customHooks";
+import {useBrowseFiles, useMediaDirectory} from "../hooks/customHooks";
 import {DirectoryRefreshButton, FileRowTagIcon, FilesRefreshButton} from "./Files";
 import {FilePreviewContext} from "./FilePreview";
 
@@ -15,14 +27,40 @@ function depthIndentation(path) {
     return '\xa0\xa0\xa0\xa0'.repeat(depth);
 }
 
-function Folder({folder, onFolderClick, sortData, selectedPath, onFileClick, onSelect}) {
+export function pathDirectory(path) {
+    // Return the full path of the path's directory.
+    // pathDirectory('/some/directory/file.txt') -> '/some/directory'
+    return path.substring(0, path.lastIndexOf('/'));
+}
+
+export function pathName(path) {
+    // Return the file's name.
+    // pathName('/some/directory/file.txt') -> 'file.txt'
+    // pathName('/some/directory') -> 'directory'
+    if (path.endsWith('/')) {
+        // Path is a directory, return its name.
+        path = path.substring(0, path.length - 1);
+    }
+    return path.substring(path.lastIndexOf('/') + 1);
+}
+
+export function splitPathParentAndName(path) {
+    if (path.endsWith('/')) {
+        // Path is a directory, return its name.
+        path = path.substring(0, path.length - 1);
+    }
+    const slashIndex = path.lastIndexOf('/');
+    return [path.substring(0, slashIndex), path.substring(slashIndex + 1, path.length)];
+}
+
+function Folder({folder, onFolderClick, sortData, selectedPaths, onFileClick, onSelect, disabled}) {
     // Creates a single table row for a folder, or a row for itself and indented rows for its children.
     let {path, children, is_empty} = folder;
     const pathWithNoTrailingSlash = path.substring(0, path.length - 1);
     const name = path.substring(pathWithNoTrailingSlash.lastIndexOf('/') + 1);
-    const f = <TableRow key={path}>
+    const f = <TableRow key={path} disabled={disabled}>
         <TableCell collapsing>
-            <Checkbox checked={selectedPath === folder['path']} onChange={() => onSelect(folder['path'])}/>
+            <Checkbox checked={selectedPaths.indexOf(folder['path']) >= 0} onChange={() => onSelect(folder['path'])}/>
         </TableCell>
         <TableCell onClick={() => onFolderClick(path)} className='file-path' colSpan={2} disabled={is_empty}>
             {depthIndentation(pathWithNoTrailingSlash)}
@@ -37,7 +75,7 @@ function Folder({folder, onFolderClick, sortData, selectedPath, onFileClick, onS
         _.forEach(children, (p, k) => {
             childPaths = [...childPaths,
                 <Path key={k} path={p} onFolderClick={onFolderClick} sortData={sortData} onFileClick={onFileClick}
-                      selectedPath={selectedPath} onSelect={onSelect}/>];
+                      selectedPaths={selectedPaths} onSelect={onSelect} disabled={disabled}/>];
         });
         return <React.Fragment>
             {f}
@@ -49,19 +87,18 @@ function Folder({folder, onFolderClick, sortData, selectedPath, onFileClick, onS
     }
 }
 
-function File({file, onFileClick, selectedPath, onSelect}) {
+function File({file, onFileClick, selectedPaths, onSelect, disabled}) {
     const {path, size} = file;
-    const name = path.substring(path.lastIndexOf('/') + 1);
-    return <TableRow key={path}>
+    return <TableRow key={path} disabled={disabled}>
         <TableCell collapsing>
-            <Checkbox checked={selectedPath === path} onChange={() => onSelect(path)}/>
+            <Checkbox checked={selectedPaths.indexOf(path) >= 0} onChange={() => onSelect(path)}/>
         </TableCell>
         <TableCell onClick={() => onFileClick(file)} className='file-path'>
             {depthIndentation(path)}
             <FileRowTagIcon file={file}/>
             {/*null size to make icon the correct size*/}
             <FileIcon file={file} size={null}/>
-            {name}
+            {pathName(path)}
         </TableCell>
         <TableCell collapsing textAlign='right'>
             {humanFileSize(size)}
@@ -69,31 +106,37 @@ function File({file, onFileClick, selectedPath, onSelect}) {
     </TableRow>
 }
 
-function Path({path, onFolderClick, onFileClick, sortData, selectedPath, onSelect}) {
+function Path({path, onFolderClick, onFileClick, sortData, selectedPaths, onSelect, disabled}) {
     if (path['path'].endsWith('/')) {
         return <Folder
             folder={path}
             onFolderClick={onFolderClick}
             sortData={sortData}
-            selectedPath={selectedPath}
+            selectedPaths={selectedPaths}
             onSelect={onSelect}
             onFileClick={onFileClick}
+            disabled={disabled}
         />
     } else {
         return <File
             file={path}
             onFileClick={onFileClick}
-            selectedPath={selectedPath}
+            selectedPaths={selectedPaths}
             onSelect={onSelect}
+            disabled={disabled}
         />
     }
 }
 
 export function FileBrowser() {
-    const [selectedPath, setSelectedPath] = React.useState(null);
+    const [selectedPaths, setSelectedPaths] = React.useState([]);
     const [deleteOpen, setDeleteOpen] = React.useState(false);
-    const [fileSelected, setFileSelected] = React.useState(false);
-    const [folderSelected, setFolderSelected] = React.useState(false);
+    const [renameOpen, setRenameOpen] = React.useState(false);
+    const [moveOpen, setMoveOpen] = React.useState(false);
+    const [makeDirectoryOpen, setMakeDirectoryOpen] = React.useState(false);
+    const [pending, setPending] = React.useState(false);
+
+    const selectedPathsCount = selectedPaths ? selectedPaths.length : 0;
 
     const {browseFiles, openFolders, setOpenFolders, fetchFiles} = useBrowseFiles();
 
@@ -113,53 +156,97 @@ export function FileBrowser() {
     }, []);
 
     const onSelect = (path) => {
-        if (path.endsWith('/') && selectedPath === path) {
-            // Deselect folder.
-            setFileSelected(false);
-            setFolderSelected(false);
-            setSelectedPath(null);
-        } else if (path.endsWith('/')) {
-            // Select folder.
-            setFileSelected(false);
-            setFolderSelected(true);
-            setSelectedPath(path);
-        } else if (selectedPath === path) {
-            // Deselect file.
-            setFileSelected(false);
-            setFolderSelected(false);
-            setSelectedPath(null);
+        let newSelectedPaths;
+        // Select or deselect if the paths was previously selected.
+        if (selectedPathsCount && selectedPaths.indexOf(path) >= 0) {
+            newSelectedPaths = selectedPaths.filter(i => i !== path);
         } else {
-            // Select file.
-            setFileSelected(true);
-            setFolderSelected(false);
-            setSelectedPath(path);
+            newSelectedPaths = [...selectedPaths, path];
         }
+        setSelectedPaths(newSelectedPaths);
+        console.debug('selectedPaths', newSelectedPaths);
     }
 
     const onDelete = async () => {
-        await deleteFile(selectedPath);
-        setDeleteOpen(false);
-        await fetchFiles();
+        await deleteFile(selectedPaths);
+        await reset();
     };
 
+    const handleMakeDirectory = async () => {
+        setMakeDirectoryOpen(false);
+        await reset();
+    }
+
+    const reset = async () => {
+        setDeleteOpen(false);
+        setSelectedPaths([]);
+        setPending(true);
+        try {
+            await fetchFiles();
+        } finally {
+            setPending(false);
+        }
+    }
+
     const footer = <TableFooter fullWidth>
         <TableRow>
             <TableHeaderCell colSpan={3}>
-                {folderSelected ? <DirectoryRefreshButton directory={selectedPath}/> : <FilesRefreshButton/>}
-                <Button
+                {selectedPathsCount ?
+                    <DirectoryRefreshButton paths={selectedPaths}/>
+                    : <FilesRefreshButton/>
+                }
+                <Button icon='trash'
                         color='red'
                         onClick={() => setDeleteOpen(true)}
-                    disabled={!fileSelected}
-                >
-                    <Icon name='close'/>
-                    Delete
-                </Button>
+                        disabled={selectedPathsCount === 0}
+                />
                 <Confirm
                     open={deleteOpen}
                     content='Are you sure you want to delete these files?'
                     onConfirm={onDelete}
                     onCancel={() => setDeleteOpen(false)}
                 />
+                <Button icon='text cursor'
+                        color='yellow'
+                        onClick={() => setRenameOpen(true)}
+                        disabled={selectedPathsCount !== 1}
+                />
+                {selectedPathsCount === 1 ?
+                    <RenameModal
+                        open={renameOpen}
+                        onClose={() => setRenameOpen(false)}
+                        onPending={setPending}
+                        path={selectedPaths[0]}
+                        onSubmit={reset}
+                    /> : null}
+                <Button icon='move'
+                        color='violet'
+                        disabled={selectedPathsCount === 0}
+                        onClick={() => setMoveOpen(true)}
+                />
+                {selectedPathsCount > 0 ?
+                    <MoveModal open={moveOpen}
+                               onClose={() => setMoveOpen(false)}
+                               paths={selectedPaths}
+                               onSubmit={reset}
+                    /> : null}
+                <Button
+                    color='blue'
+                    onClick={() => setMakeDirectoryOpen(true)}
+                    disabled={selectedPaths && selectedPaths.length > 1}
+                    style={{paddingLeft: '1em', paddingRight: '0.8em'}}
+                >
+                    <IconGroup>
+                        <Icon name='folder'/>
+                        <Icon corner name='add'/>
+                    </IconGroup>
+                </Button>
+                <MakeDirectoryModal
+                    open={makeDirectoryOpen}
+                    onClose={() => setMakeDirectoryOpen(false)}
+                    parent={selectedPaths.length ? selectedPaths[0] : null}
+                    onSubmit={handleMakeDirectory}
+                />
             </TableHeaderCell>
         </TableRow>
     </TableFooter>;
@@ -197,10 +284,187 @@ export function FileBrowser() {
                 onFolderClick={onFolderClick}
                 onFileClick={(i) => setPreviewFile(i)}
                 sortData={sortData}
-                selectedPath={selectedPath}
+                selectedPaths={selectedPaths}
                 onSelect={onSelect}
+                disabled={pending}
             />}
             footer={footer}
         />
     </>
 }
+
+export function RenameModal({open, onClose, path, onSubmit, onPending}) {
+    const [parent, name] = splitPathParentAndName(path);
+    const [value, setValue] = React.useState(name);
+    const disabled = value === name;
+
+    const handleInputChange = (e, props) => {
+        if (e) {
+            e.preventDefault();
+        }
+        setValue(props.value);
+    }
+
+    const handleSubmit = async (e) => {
+        if (e) {
+            e.preventDefault();
+        }
+        try {
+            console.log(`Renaming: ${path} to ${value}`);
+            onPending(true);
+            await renamePath(path, value);
+        } finally {
+            onClose();
+            onPending(false);
+            if (onSubmit) {
+                await onSubmit();
+            }
+        }
+    }
+
+    useEffect(() => {
+        setValue(pathName(path));
+    }, [path]);
+
+    return <Modal closeIcon
+                  open={open}
+                  onClose={onClose}
+    >
+        <ModalHeader>Rename: {name}</ModalHeader>
+        <ModalContent>
+            <Header as='h4'>New Name</Header>
+            <pre>{parent ? parent + '/' : null}{value}</pre>
+
+            <Input fluid
+                   label='Name'
+                   value={value}
+                   onChange={handleInputChange}
+            />
+        </ModalContent>
+        <ModalActions>
+            <Button
+                color='violet'
+                disabled={disabled}
+                onClick={handleSubmit}
+            >
+                Rename
+            </Button>
+            <Button
+                onClick={() => setValue(name)}
+                floated='left'
+            >
+                Reset
+            </Button>
+        </ModalActions>
+    </Modal>
+}
+
+export function MoveModal({open, paths, onClose, onSubmit}) {
+    const [destination, setDestination] = React.useState('');
+    const [pending, setPending] = React.useState(false);
+
+    const handleDirectory = (value) => {
+        setDestination(value);
+    }
+
+    const handleMove = async () => {
+        try {
+            setPending(true);
+            await movePaths(destination, paths);
+            onClose();
+        } catch (e) {
+            console.error(e);
+            setPending(false);
+        }
+        if (onSubmit) {
+            await onSubmit();
+        }
+    }
+
+    const fileCell = i => <TableCell>
+        <pre style={{marginTop: 0, marginBottom: 0}}>{i}</pre>
+    </TableCell>;
+
+    const destinationRows = paths.map(i => <TableRow key={i}>
+        {fileCell(i)}
+        {destination ? fileCell(destination + '/' + pathName(i)) : fileCell(pathName(i))}
+    </TableRow>);
+
+    return <Modal closeIcon
+                  onClose={onClose}
+                  open={open}
+    >
+        <ModalHeader>Move Files/Directories</ModalHeader>
+        <ModalContent>
+            <p>Search for a directory to move your files into:</p>
+            <DirectorySearch onSelect={handleDirectory} disabled={pending}/>
+
+            <Table>
+                <TableHeader>
+                    <TableRow>
+                        <TableHeaderCell>Source</TableHeaderCell>
+                        <TableHeaderCell>Destination</TableHeaderCell>
+                    </TableRow>
+                </TableHeader>
+                <TableBody>
+                    {destinationRows}
+                </TableBody>
+            </Table>
+        </ModalContent>
+        <ModalActions>
+            <Button
+                color='violet'
+                onClick={handleMove}
+                disabled={pending}
+            >Move</Button>
+            <Button
+                onClick={() => setDestination(null)}
+                floated='left'
+                disabled={pending}
+            >Reset</Button>
+        </ModalActions>
+    </Modal>
+}
+
+export function MakeDirectoryModal({open, onClose, parent, onSubmit}) {
+    const [value, setValue] = useState('');
+    const mediaDirectory = useMediaDirectory();
+
+    const handleInputChange = (e, props) => {
+        if (e) {
+            e.preventDefault();
+        }
+        setValue(props.value);
+    }
+
+    const handleSubmit = async (e) => {
+        if (e) {
+            e.preventDefault();
+        }
+        try {
+            const path = parent ? `${parent}/${value}` : value;
+            await makeDirectory(path);
+            setValue('');
+            if (onSubmit) {
+                await onSubmit();
+            }
+        } catch (e) {
+            console.error(e);
+        }
+    }
+
+    return <Modal closeIcon open={open} onClose={onClose}>
+        <ModalHeader>Make Directory</ModalHeader>
+        <ModalContent>
+            <pre>{mediaDirectory}/{parent}{value}</pre>
+            <Input fluid
+                   label='Name'
+                   value={value}
+                   onChange={handleInputChange}
+            />
+        </ModalContent>
+        <ModalActions>
+            <Button color='violet' onClick={handleSubmit}>Create</Button>
+        </ModalActions>
+    </Modal>
+}

app/src/components/FilePreview.js

@@ -1,12 +1,13 @@
 import React from "react";
 import _ from "lodash";
 import {Grid, Header as SHeader, Image,} from "semantic-ui-react";
-import {TagsProvider, TagsSelector} from "../Tags";
+import {TagsSelector} from "../Tags";
 import {Media} from "../contexts/contexts";
 import {encodeMediaPath} from "./Common";
-import {addTag, fetchFile, removeTag} from "../api";
+import {fetchFile, tagFileGroup, untagFileGroup} from "../api";
 import {StlViewer} from "react-stl-viewer";
 import {Button, Modal, ModalActions, ModalContent, ModalHeader} from "./Theme";
+import {toast} from "react-semantic-toasts-2";
 
 function getMediaPathURL(previewFile) {
     if (previewFile['primary_path']) {
@@ -26,9 +27,9 @@ function getDownloadPathURL(previewFile) {
 
 function getEpubViewerURL(previewFile) {
     if (previewFile['primary_path']) {
-        return `/epub.html?url=download/${encodeMediaPath(previewFile['primary_path'])}`;
+        return `/epub/epub.html?url=download/${encodeMediaPath(previewFile['primary_path'])}`;
     } else {
-        return `/epub.html?url=download/${encodeMediaPath(previewFile['path'])}`;
+        return `/epub/epub.html?url=download/${encodeMediaPath(previewFile['path'])}`;
     }
 }
 
@@ -65,7 +66,7 @@ function getImageModal(previewFile) {
 
 function getEpubModal(previewFile) {
     const downloadURL = getDownloadPathURL(previewFile);
-    const viewerUrl = `/epub.html?url=${downloadURL}`;
+    const viewerUrl = `/epub/epub.html?url=${downloadURL}`;
     return <ModalContent>
         <div className='full-height'>
             <iframe title='textModal' src={viewerUrl}
@@ -138,6 +139,15 @@ function getSTLModal(previewFile) {
     </React.Fragment>
 }
 
+function getGenericModal(previewFile) {
+    const path = previewFile.primary_path ?? previewFile.path;
+    const name = path.replace(/^.*[\\\/]/, '');
+
+    return <React.Fragment>
+        <ModalHeader>{name}</ModalHeader>
+    </React.Fragment>
+}
+
 export const FilePreviewContext = React.createContext({
     previewFile: null,
     setPreviewFile: null,
@@ -183,29 +193,28 @@ export function FilePreviewProvider({children}) {
     }
 
     const localAddTag = async (name) => {
-        await addTag(previewFile, name);
+        await tagFileGroup(previewFile, name);
         await localFetchFile();
     }
 
     const localRemoveTag = async (name) => {
-        await removeTag(previewFile, name);
+        await untagFileGroup(previewFile, name);
         await localFetchFile();
     }
 
-    function setModalContent(content, url, downloadURL) {
-        let openButton;
-        if (url) {
-            openButton = <Button color='blue' as='a' href={url}>Open</Button>;
-        }
+    function setModalContent(content, url, downloadURL, path) {
+        const openButton = url ? <Button color='blue' as='a' href={url}>Open</Button> : null;
         const closeButton = <Button onClick={handleClose}>Close</Button>;
         const tagsDisplay = <TagsSelector selectedTagNames={previewFile['tags']} onAdd={localAddTag}
                                           onRemove={localRemoveTag}/>;
-        let downloadButton;
-        if (downloadURL) {
-            downloadButton = <Button color='yellow' as='a' href={downloadURL} floated='left'>
-                Download
-            </Button>;
-        }
+        const downloadButton = downloadURL ?
+            <Button color='yellow' as='a' href={downloadURL} floated='left'>Download</Button>
+            : null;
+        const pathContent = path ? <ModalContent>
+                <pre>{path}</pre>
+            </ModalContent>
+            : null;
+        console.log('path', path);
 
         setPreviewModal(<Modal closeIcon
                                size='fullscreen'
@@ -213,6 +222,7 @@ export function FilePreviewProvider({children}) {
                                onClose={e => handleClose(e)}
         >
             {content}
+            {pathContent}
             <ModalActions>
                 <Media at='mobile'>
                     <Grid>
@@ -244,41 +254,47 @@ export function FilePreviewProvider({children}) {
         setPreviewModal(null);
         if (previewFile && !_.isObject(previewFile)) {
             console.error(`Unknown previewFile type: ${typeof previewFile}`);
+            toast({
+                type: 'error',
+                title: 'Cannot preview file',
+                description: 'Cannot preview file',
+                time: 5000,
+            });
             return;
         }
 
         if (previewFile && !_.isEmpty(previewFile)) {
             const {mimetype, size} = previewFile;
             const path = previewFile['primary_path'] || previewFile['path'];
+            console.debug('Previewing file', previewFile);
             const lowerPath = path.toLowerCase();
-            console.debug(`useFilePreview path=${previewFile['path']} mimetype=${mimetype}`);
             const url = getMediaPathURL(previewFile);
             const downloadURL = getDownloadPathURL(previewFile);
             if (mimetype.startsWith('text/') && size > MAXIMUM_TEXT_SIZE) {
                 // Large text files should be downloaded.
                 window.open(downloadURL);
             } else if (mimetype.startsWith('text/') || mimetype.startsWith('application/json')) {
-                setModalContent(getIframeModal(previewFile), url, downloadURL);
+                setModalContent(getIframeModal(previewFile), url, downloadURL, path);
             } else if (mimetype.startsWith('video/')) {
-                setModalContent(getVideoModal(previewFile), url, downloadURL);
+                setModalContent(getVideoModal(previewFile), url, downloadURL, path);
             } else if (mimetype.startsWith('audio/')) {
-                setModalContent(getAudioModal(previewFile), url, downloadURL);
+                setModalContent(getAudioModal(previewFile), url, downloadURL, path);
             } else if (mimetype.startsWith('application/epub')) {
                 const viewerURL = getEpubViewerURL(previewFile);
-                setModalContent(getEpubModal(previewFile), viewerURL, downloadURL);
+                setModalContent(getEpubModal(previewFile), viewerURL, downloadURL, path);
             } else if (mimetype.startsWith('application/pdf')) {
-                setModalContent(getIframeModal(previewFile), url, downloadURL);
+                setModalContent(getIframeModal(previewFile), url, downloadURL, path);
             } else if (mimetype.startsWith('image/')) {
-                setModalContent(getImageModal(previewFile), url);
+                setModalContent(getImageModal(previewFile), url, null, path);
             } else if (mimetype.startsWith('model/stl')) {
-                setModalContent(getSTLModal(previewFile), null, downloadURL);
+                setModalContent(getSTLModal(previewFile), null, downloadURL, path);
             } else if (mimetype.startsWith('application/octet-stream') && lowerPath.endsWith('.mp3')) {
-                setModalContent(getAudioModal(previewFile), url, downloadURL);
+                setModalContent(getAudioModal(previewFile), url, downloadURL, path);
             } else if (mimetype.startsWith('application/octet-stream') && lowerPath.endsWith('.stl')) {
-                setModalContent(getSTLModal(previewFile), null, downloadURL);
+                setModalContent(getSTLModal(previewFile), null, downloadURL, path);
             } else {
                 // No special handler for this file type, just open it.
-                window.open(downloadURL);
+                setModalContent(getGenericModal(previewFile), url, downloadURL, path);
             }
         }
     }, [previewFile]);

app/src/components/Files.js

@@ -9,7 +9,6 @@ import {
     Dropdown,
     Form,
     Image,
-    Label,
     PlaceholderLine,
     TableCell,
     TableHeaderCell,
@@ -28,6 +27,7 @@ import {
     PageContainer,
     Paginator,
     PreviewLink,
+    TagIcon,
     textEllipsis,
     useTitle
 } from "./Common";
@@ -40,7 +40,7 @@ import {
     useSearchView
 } from "../hooks/customHooks";
 import {Route, Routes} from "react-router-dom";
-import {CardPlacholder} from "./Placeholder";
+import {CardPlaceholder} from "./Placeholder";
 import {ArchiveCard, ArchiveRowCells} from "./Archive";
 import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
 import {Media, StatusContext, ThemeContext} from "../contexts/contexts";
@@ -72,7 +72,7 @@ function EbookCard({file}) {
 
     const downloadUrl = `/download/${encodeMediaPath(file.primary_path)}`;
     const isEpub = file['mimetype'].startsWith('application/epub');
-    const viewerUrl = isEpub ? `/epub.html?url=${downloadUrl}` : null;
+    const viewerUrl = isEpub ? `/epub/epub.html?url=${downloadUrl}` : null;
 
     const color = mimetypeColor(file.mimetype);
     return <Card color={color}>
@@ -125,7 +125,7 @@ function FileCard({file}) {
         file.mimetype.startsWith('application/epub') || file.mimetype.startsWith('application/x-mobipocket-ebook')
     );
 
-    if (file.model === 'video') {
+    if (file.model === 'video' && 'video' in file) {
         return <VideoCard key={file['primary_path']} file={file}/>;
     } else if (file.model === 'archive') {
         return <ArchiveCard key={file['primary_path']} file={file}/>;
@@ -167,7 +167,7 @@ export function FileCards({files}) {
     } else if (files && files.length === 0) {
         return <Segment>No results!</Segment>
     } else {
-        return <CardGroupCentered><CardPlacholder/></CardGroupCentered>
+        return <CardGroupCentered><CardPlaceholder/></CardGroupCentered>
     }
 }
 
@@ -269,9 +269,7 @@ export function FileTable({files, selectOn, onSelect, footer, selectedKeys}) {
 
 export function FileRowTagIcon({file}) {
     if (file.tags && file.tags.length) {
-        return <Label circular color='green' style={{padding: '0.5em', marginRight: '0.5em'}}>
-            <Icon name='tag' style={{margin: 0}}/>
-        </Label>;
+        return <TagIcon/>;
     }
 }
 
@@ -431,10 +429,10 @@ export function TagsQuerySelector({onChange}) {
     const {searchParams, updateQuery} = useQuery();
     const activeTags = searchParams.getAll('tag');
 
-    const localOnChange = (name) => {
-        updateQuery({'tag': name});
+    const localOnChange = (tagNames) => {
+        updateQuery({'tag': tagNames});
         if (onChange) {
-            onChange(name);
+            onChange(tagNames);
         }
     }
 
@@ -464,16 +462,16 @@ export function FilesSearchView({
         {key: 'model', text: '3D Model', value: 'model'},
     ];
 
-    const {searchFiles, totalPages, activePage, setPage} = useSearchFiles(24, emptySearch, model);
+    const {searchFiles, pages} = useSearchFiles(24, emptySearch, model);
 
     const {body, paginator, selectButton, viewButton, limitDropdown, tagQuerySelector} = FilesView(
         searchFiles,
-        activePage,
-        totalPages,
+        pages.activePage,
+        pages.totalPages,
         null,
         null,
         null,
-        setPage,
+        pages.setPage,
         true,
     );
 
@@ -597,7 +595,7 @@ export function FilesRefreshButton() {
     </Button>;
 }
 
-export function DirectoryRefreshButton({directory}) {
+export function DirectoryRefreshButton({paths}) {
     const {status} = useContext(StatusContext);
     const refreshing = status && status['flags'] && status['flags'].indexOf('refreshing') >= 0;
     const refreshingDirectory = status && status['flags'] && status['flags'].indexOf('refreshing_directory') >= 0;
@@ -606,7 +604,7 @@ export function DirectoryRefreshButton({directory}) {
 
     const handleClick = async () => {
         setLoading(true);
-        await refreshDirectoryFiles(directory);
+        await refreshDirectoryFiles(paths);
     }
 
     // Clear loading when global refresh event completes.
@@ -620,6 +618,6 @@ export function DirectoryRefreshButton({directory}) {
                    disabled={loading || refreshing || refreshingDirectory}
     >
         <Icon name='refresh'/>
-        Refresh Directory
+        Refresh
     </Button>
 }

app/src/components/Headline.js

@@ -8,14 +8,14 @@ import _ from "lodash";
 import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
 import {FileRowTagIcon} from "./Files";
 
-const HeadlineParagraph = ({headline}) => {
+export const HeadlineText = ({headline, openTag = '<u>', closeTag = '</u>'}) => {
     const {s} = React.useContext(ThemeContext);
     if (headline) {
-        headline = headline.replaceAll('<b>', '<b><u>');
-        headline = headline.replaceAll('</b>', '</b></u>');
+        headline = headline.replaceAll('<b>', openTag);
+        headline = headline.replaceAll('</b>', closeTag);
         headline = headline.replaceAll('\n', '<br/>');
         // Postgres inserts <b>...</b> tags which we will use.
-        return <p {...s} dangerouslySetInnerHTML={{__html: headline}}></p>
+        return <span {...s} dangerouslySetInnerHTML={{__html: headline}}></span>
     }
 }
 
@@ -33,18 +33,18 @@ const FileHeadline = ({file, to}) => {
 
     let header = title || name;
     if (title_headline) {
-        header = <PreviewLink file={file}><HeadlineParagraph headline={title_headline}/></PreviewLink>;
+        header = <PreviewLink file={file}><HeadlineText headline={title_headline}/></PreviewLink>;
     }
     if (to) {
-        header = <Link to={to}><HeadlineParagraph headline={title_headline}/></Link>
+        header = <Link to={to}><HeadlineText headline={title_headline}/></Link>
     }
 
     let body = <p><small>No highlights available.</small></p>;
     if (b_headline || c_headline || d_headline) {
         body = <>
-            <HeadlineParagraph headline={b_headline}/>
-            <HeadlineParagraph headline={c_headline}/>
-            <HeadlineParagraph headline={d_headline}/>
+            <HeadlineText headline={b_headline}/>
+            <HeadlineText headline={c_headline}/>
+            <HeadlineText headline={d_headline}/>
         </>;
     }
 

app/src/components/Map.js

@@ -71,8 +71,8 @@ function DownloadMessage() {
             </p>
 
             <p><b>Download only the areas you need</b>. Large regions like all of Asia, or the entire
-                planet are most likely <b>too large</b> and won't render quickly. I recommend only
-                importing files less than 1GB on a Raspberry Pi.</p>
+                planet are most likely <b>too large</b> and won't render quickly. It is recommend to only
+                import files less than 1GB on a Raspberry Pi.</p>
 
             <p>Only <b>*.osm.pbf</b> files are supported!</p>
 
@@ -235,7 +235,6 @@ class ManageMap extends React.Component {
         return <PageContainer>
             <WROLModeMessage content='Cannot modify Map'/>
             <DockerMapImportWarning/>
-            <DownloadMessage/>
             <SlowImportMessage/>
             <Loader size='large' active={import_running} inline='centered'>
                 {importingMessage}
@@ -263,6 +262,7 @@ class ManageMap extends React.Component {
                     </TableRow>
                 </TableFooter>
             </Table>
+            <DownloadMessage/>
         </PageContainer>
     }
 }

app/src/components/Nav.js

@@ -10,11 +10,12 @@ const links = [
     {text: 'Videos', to: '/videos', key: 'videos'},
     {text: 'Archive', to: '/archive', key: 'archive'},
     {text: 'Map', to: '/map', key: 'map'},
+    {text: 'Zim', to: '/zim', key: 'zim'},
     {text: 'Files', to: '/files', key: 'files'},
-    {text: 'Inventory', to: '/inventory', key: 'inventory'},
     {
         text: 'More', key: 'more',
         links: [
+            {text: 'Inventory', to: '/inventory', key: 'inventory'},
             {to: '/more/otp', text: 'One Time Pad', end: true},
             {to: '/more/statistics', text: 'Statistics', end: true},
             {to: '/more/vin', text: 'VIN Decoder', end: true},

app/src/components/Placeholder.js

@@ -3,7 +3,7 @@ import React, {useContext} from "react";
 import {ThemeContext} from "../contexts/contexts";
 import {Card, Placeholder} from "./Theme";
 
-export function CardPlacholder() {
+export function CardPlaceholder() {
     const {s} = useContext(ThemeContext);
 
     return <Card>
@@ -22,7 +22,7 @@ export function CardPlacholder() {
 
 export function VideoPlaceholder() {
     return <CardGroup doubling stackable>
-        <CardPlacholder/>
+        <CardPlaceholder/>
     </CardGroup>
 }
 
@@ -39,3 +39,11 @@ export function ProgressPlaceholder() {
         <PlaceholderLine/>
     </Placeholder>
 }
+
+export function TextPlaceholder() {
+    return <Placeholder style={{marginBottom: '1em'}}>
+        <PlaceholderLine/>
+        <PlaceholderLine/>
+        <PlaceholderLine/>
+    </Placeholder>
+}

app/src/components/Search.js

@@ -0,0 +1,113 @@
+import React, {useEffect, useState} from "react";
+import {createSearchParams, Route, Routes, useHref, useNavigate} from "react-router-dom";
+import {FilesSearchView} from "./Files";
+import {usePages, useQuery} from "../hooks/customHooks";
+import {ZimSearchView} from "./Zim";
+import {searchEstimate} from "../api";
+import {normalizeEstimate, TabLinks} from "./Common";
+
+export const useSearch = (defaultLimit = 48, totalPages = 0, emptySearch = false, model) => {
+    const navigate = useNavigate();
+    const href = useHref();
+
+    const {searchParams, updateQuery} = useQuery();
+    const searchStr = searchParams.get('q');
+    const activeTags = searchParams.getAll('tag');
+    const pages = usePages(defaultLimit, totalPages);
+    const filter = searchParams.get('filter');
+    const model_ = searchParams.get('model') || model;
+
+    const setSearchStr = (value) => {
+        navigate({
+            pathname: '/search',
+            // Start new search at offset 0.
+            search: createSearchParams({q: value, o: 0}).toString(),
+        });
+    }
+
+    const setTags = (tags) => {
+        updateQuery({tag: tags});
+    }
+
+    const addTag = (name) => {
+        const newTags = [...activeTags, name];
+        setTags(newTags);
+    }
+
+    const removeTag = (name) => {
+        const newTags = activeTags.filter(i => i !== name);
+        setTags(newTags);
+    }
+
+    return {
+        activeTags,
+        addTag,
+        filter,
+        href,
+        model: model_,
+        pages,
+        removeTag,
+        searchParams,
+        searchStr,
+        setSearchStr,
+        setTags,
+    }
+}
+
+
+export const useSearchEstimate = (search_str, activeTags) => {
+    const [fileGroups, setFileGroups] = useState('?');
+    const [zims, setZims] = useState(null);
+    const [zimsSum, setZimsSum] = useState('?');
+
+    const localFetchSearchEstimate = async () => {
+        if (!search_str && !activeTags) {
+            return;
+        }
+        try {
+            const estimates = await searchEstimate(search_str, activeTags);
+            console.debug('estimates', estimates);
+            setFileGroups(estimates['file_groups']);
+            setZimsSum(estimates['zimSum']);
+            setZims(estimates['zims']);
+        } catch (e) {
+            console.error('Failed to fetch estimates');
+            console.error(e);
+            setFileGroups(null);
+            setZimsSum(null);
+            setZims(null);
+        }
+    }
+
+    useEffect(() => {
+        localFetchSearchEstimate();
+    }, [search_str, JSON.stringify(activeTags)]);
+
+    return {fileGroups, zimsSum, zims}
+}
+
+export function SearchView() {
+    const {searchStr, activeTags} = useSearch();
+    const estimates = useSearchEstimate(searchStr, activeTags);
+
+    let filesTabName = 'Files';
+    let zimsTabName = 'Zims';
+    if (estimates) {
+        const {fileGroups, zimsSum} = estimates;
+        filesTabName = `Files (${normalizeEstimate(fileGroups)})`;
+        zimsTabName = `Zims (${normalizeEstimate(zimsSum)})`;
+    }
+
+    const links = [
+        {text: filesTabName, to: '/search', key: 'filesSearch', end: true},
+        {text: zimsTabName, to: '/search/zim', key: 'zimsSearch'},
+    ]
+
+    return <React.Fragment>
+        <TabLinks links={links}/>
+        <Routes>
+            <Route path='/*' element={<FilesSearchView/>}/>
+            <Route path='/zim' exact element={<ZimSearchView estimates={estimates}/>}/>
+        </Routes>
+    </React.Fragment>
+}

app/src/components/SortableTable.js

@@ -6,7 +6,10 @@ import {Table} from "./Theme";
 export class SortableTable extends React.Component {
     constructor(props) {
         super(props);
-        this.state = {sortColumn: null, direction: 'ascending'};
+        this.state = {
+            sortColumn: props.defaultSortColumn || null,
+            direction: props.defaultDirection || 'ascending',
+        };
     }
 
     changeSort = (key) => {

app/src/components/Theme.js

@@ -186,8 +186,8 @@ export function Divider({...props}) {
 }
 
 export function Header(props) {
-    const {t, inverted} = useContext(ThemeContext);
-    props = defaultGrey({...t, ...props}, inverted);
+    const {t} = useContext(ThemeContext);
+    props = {...t, ...props};
     return <SHeader {...props}/>
 }
 

app/src/components/VideoPlayer.js

@@ -1,5 +1,5 @@
 import React, {useContext, useState} from 'react';
-import {addTag, deleteVideos, removeTag} from "../api";
+import {tagFileGroup, deleteVideos, untagFileGroup} from "../api";
 import {Link, useNavigate, useParams} from "react-router-dom";
 import _ from "lodash";
 import {
@@ -159,12 +159,12 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
     const tabMenu = theme === darkTheme ? {inverted: true, attached: true} : {attached: true};
 
     const localAddTag = async (name) => {
-        await addTag(videoFile, name);
+        await tagFileGroup(videoFile, name);
         await fetchVideo();
     }
 
     const localRemoveTag = async (name) => {
-        await removeTag(videoFile, name);
+        await untagFileGroup(videoFile, name);
         await fetchVideo();
     }
 

app/src/components/Videos.js

@@ -113,17 +113,15 @@ function VideosPage() {
     }
 
     const onSelect = (path, checked) => {
-        if (checked && path) {
-            setSelectedVideos([...selectedVideos, path]);
-        } else if (path) {
-            setSelectedVideos(selectedVideos.filter(i => i !== path));
-        }
+        const newSelectedVideos = checked && path ? [...selectedVideos, path] : selectedVideos.filter(i => i !== path);
+        console.debug(`Selected Videos: ${newSelectedVideos}`);
+        setSelectedVideos(newSelectedVideos);
     }
 
     const onDelete = async (e) => {
         e.preventDefault();
         setDeleteOpen(false);
-        const videoIds = videos.filter(i => selectedVideos.indexOf(i['path']) >= 0).map(i => i['video']['id']);
+        const videoIds = videos.filter(i => selectedVideos.indexOf(i['primary_path']) >= 0).map(i => i['video']['id']);
         await deleteVideos(videoIds);
         await fetchVideos();
         setSelectedVideos([]);

app/src/components/Zim.js

@@ -0,0 +1,548 @@
+import {useOutdatedZims, useSearchZim} from "../hooks/customHooks";
+import {
+    Accordion,
+    Button,
+    Divider,
+    Header,
+    Icon,
+    Modal,
+    ModalActions,
+    ModalContent,
+    ModalHeader,
+    Placeholder,
+    Segment
+} from "./Theme";
+import {HeadlineText} from "./Headline";
+import {TextPlaceholder} from "./Placeholder";
+import {
+    AccordionContent,
+    AccordionTitle,
+    Button as SButton,
+    Dropdown,
+    Icon as SIcon,
+    Label,
+    Message,
+    PlaceholderHeader,
+    PlaceholderLine,
+    TableCell,
+    TableFooter,
+    TableHeaderCell,
+    TableRow
+} from "semantic-ui-react";
+import React, {useContext, useState} from "react";
+import {
+    encodeMediaPath,
+    humanFileSize,
+    normalizeEstimate,
+    PageContainer,
+    Paginator,
+    TabLinks,
+    TagIcon,
+    useTitle
+} from "./Common";
+import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
+import {TagsSelector} from "../Tags";
+import {
+    deleteOutdatedZims,
+    fetchZims,
+    fetchZimSubscriptions,
+    refreshFiles,
+    saveSettings,
+    tagZimEntry,
+    untagZimEntry,
+    zimSubscribe,
+    zimUnsubscribe
+} from "../api";
+import {useSearch} from "./Search";
+import {TagsQuerySelector} from "./Files";
+import {ThemeContext} from "../contexts/contexts";
+import {Link, Route, Routes} from "react-router-dom";
+import {SortableTable} from "./SortableTable";
+import {toast} from "react-semantic-toasts-2";
+
+const VIEWER_URL = `http://${window.location.hostname}:8085/`;
+
+export const OutdatedZimsMessage = ({onClick}) => {
+    const [open, setOpen] = React.useState(false);
+
+    const {outdated, current} = useOutdatedZims();
+
+    const onOpen = () => setOpen(true);
+    const onClose = () => setOpen(false);
+
+    const handleDelete = async (e) => {
+        if (e) {
+            e.preventDefault();
+        }
+        try {
+            const success = await deleteOutdatedZims();
+            if (success) {
+                toast({
+                    type: 'info',
+                    title: 'Zims deleted',
+                    description: 'Outdated Zims have been deleted.',
+                    time: 5000,
+                });
+            } else {
+                toast({
+                    type: 'error',
+                    title: 'Failed to delete Zims',
+                    description: 'Outdated Zims have NOT been deleted.  See server logs.',
+                    time: 5000,
+                });
+            }
+        } finally {
+            onClose();
+        }
+    }
+
+    const handleIgnore = async (e) => {
+        if (e) {
+            e.preventDefault();
+        }
+        const config = {ignore_outdated_zims: true};
+        await saveSettings(config);
+        if (onClick) {
+            await onClick();
+        }
+    }
+
+    let modalContent = <Placeholder>
+        <PlaceholderLine/>
+    </Placeholder>;
+    if (outdated && outdated.length > 0) {
+        modalContent = <>
+            <Header as='h3'>To Delete</Header>
+            {outdated.map(i => <pre key={i}>{i}</pre>)}
+
+            <Header as='h3'>To Keep</Header>
+            {current.map(i => <pre key={i}>{i}</pre>)}
+        </>
+    }
+
+    return <Message icon info>
+        <SIcon name='question'/>
+        <Message.Content>
+            <Message.Header>Outdated Zim Files</Message.Header>
+            New Zim files have been downloaded. Outdated Zim files can be removed.
+            <p></p>
+
+            <SButton primary onClick={() => setOpen(true)}>Delete</SButton>
+            <Modal closeIcon
+                   open={open}
+                   onClose={onClose}
+                   onOpen={onOpen}
+            >
+                <ModalHeader>Delete</ModalHeader>
+                <ModalContent>
+                    {modalContent}
+                </ModalContent>
+                <ModalActions>
+                    <Button color='red' floated='left' onClick={handleDelete}>Delete</Button>
+                    <Button onClick={onClose}>Close</Button>
+                </ModalActions>
+            </Modal>
+
+            <Link to='/files?folders=zims'><SButton>Delete Manually</SButton></Link>
+            <SButton secondary onClick={handleIgnore}>Ignore Forever</SButton>
+        </Message.Content>
+    </Message>
+}
+
+export const KiwixRestartMessage = () => {
+    return <Message icon warning>
+        <SIcon name='exclamation'/>
+        <Message.Content>
+            <Message.Header>Kiwix must be restarted</Message.Header>
+            <p>New Zim files have been downloaded; you must restart your containers.</p>
+
+            <p>Run the following to restart your containers:</p>
+            <pre>  docker-compose restart</pre>
+        </Message.Content>
+    </Message>
+}
+
+const ZimSearchEntry = ({zimId, onTag, onUntag, entry}) => {
+    const {path, title, headline, tag_names} = entry;
+    const [open, setOpen] = React.useState(false);
+    const url = `/api/zim/${zimId}/entry/${encodeMediaPath(path)}`;
+
+    const handleClick = async (e) => {
+        if (e) {
+            e.preventDefault();
+        }
+        setOpen(true);
+    }
+
+    const localAddTag = (name) => {
+        onTag(zimId, path, name);
+    }
+
+    const localUntag = (name) => {
+        onUntag(zimId, path, name);
+    }
+
+    const tagIcon = tag_names && tag_names.length > 0 ? <TagIcon/> : null;
+
+    return <div style={{marginLeft: '0.5em'}}>
+        <Header
+            as='h3'
+            className='clickable'
+            onClick={handleClick}
+        >
+            <u>
+                <HeadlineText headline={title || path} openTag={'<i>'} closeTag={'</i>'}/>
+            </u>
+            {tagIcon}
+        </Header>
+        <HeadlineText headline={headline}/>
+        <Modal closeIcon
+               open={open}
+               onClose={() => setOpen(false)}>
+            <ModalContent>
+                <div className='full-height'>
+                    <iframe title='textModal' src={url}
+                            style={{
+                                height: '100%', width: '100%', border: 'none', position: 'absolute', top: 0,
+                                // Use white to avoid iframe displaying with dark-theme.
+                                backgroundColor: '#ffffff',
+                            }}/>
+                </div>
+            </ModalContent>
+            <ModalActions>
+                <Grid>
+                    <Grid.Column mobile={10} tablet={14}>
+                        <TagsSelector selectedTagNames={tag_names} onAdd={localAddTag} onRemove={localUntag}/>
+                    </Grid.Column>
+                    <Grid.Column width={2}>
+                        <Button color='blue' as='a' href={url} target='_blank'>Open</Button>
+                    </Grid.Column>
+                </Grid>
+            </ModalActions>
+        </Modal>
+    </div>
+}
+
+const ZimAccordion = ({data, index, activeIndex, onClick, searchStr, activeTags}) => {
+    const {id, estimate, metadata} = data;
+    const {title, date} = metadata;
+    const {zim, fetchSearch, pages, loading} = useSearchZim(searchStr, id, index === activeIndex, activeTags);
+    const {t} = useContext(ThemeContext);
+
+    const localAddTag = async (zimId, path, name) => {
+        await tagZimEntry(zimId, path, name);
+        await fetchSearch();
+    }
+
+    const localUntag = async (zimId, path, name) => {
+        await untagZimEntry(zimId, path, name);
+        await fetchSearch();
+    }
+
+    let body = <TextPlaceholder/>;
+    if (zim && !loading) {
+        const {search} = zim;
+        if (search && search.length > 0) {
+            body = search.map(i => <Segment vertical key={i['path']}>
+                <ZimSearchEntry zimId={id} onTag={localAddTag} onUntag={localUntag} entry={i}/>
+            </Segment>);
+        } else {
+            body = <p {...t}>No results</p>;
+        }
+    }
+
+    const paginator = <center style={{marginTop: '2em'}}>
+        <Paginator activePage={pages.activePage} totalPages={pages.totalPages} onPageChange={pages.setPage}/>
+    </center>;
+
+    return <React.Fragment>
+        <AccordionTitle
+            index={index}
+            active={index === activeIndex}
+            onClick={() => onClick(index, activeIndex)}
+        >
+            <Header as='h3'>
+                <Icon name='dropdown'/> {title} <Label>{normalizeEstimate(estimate)}</Label>
+            </Header>
+        </AccordionTitle>
+        <AccordionContent active={index === activeIndex}>
+            <Header as='h4'>{date}</Header>
+            {body}
+            {paginator}
+        </AccordionContent>
+    </React.Fragment>
+}
+
+const ZimsRefreshWarning = () => {
+    return <Message icon warning onClick={refreshFiles}>
+        <SIcon name='hand point right'/>
+        <Message.Content>
+            <Message.Header>No Zims have been indexed.</Message.Header>
+            <a href='#'>Click here</a> to refresh all your files.
+        </Message.Content>
+    </Message>;
+}
+
+export const ZimSearchView = ({estimates}) => {
+    const [activeIndex, setActiveIndex] = React.useState(null);
+    const {searchStr, activeTags, setTags} = useSearch();
+    const {zims} = estimates;
+
+    const handleClick = (index, activeIndex_) => {
+        setActiveIndex(index === activeIndex_ ? -1 : index);
+    }
+
+    let body = <TextPlaceholder/>;
+    if (zims && zims.length === 0) {
+        body = <ZimsRefreshWarning/>;
+    } else if (zims) {
+        body = zims.map((i, index) => <ZimAccordion
+            key={i['path']}
+            index={index}
+            activeIndex={activeIndex}
+            data={i}
+            searchStr={searchStr}
+            activeTags={activeTags}
+            onClick={handleClick}
+        />);
+    }
+
+    return <>
+        <TagsQuerySelector onChange={setTags}/>
+        <Accordion>
+            {body}
+        </Accordion>
+    </>
+}
+
+const ViewerMessage = () => {
+    return <Message warning icon>
+        <SIcon name='warning'/>
+        <Message.Content>
+            <p>You can view your Zim files using the Kiwix app, or at <a href={VIEWER_URL}>{VIEWER_URL}</a></p>
+        </Message.Content>
+    </Message>
+}
+
+const ZimCatalogItemRow = ({item, subscriptions, iso_639_codes, fetchSubscriptions}) => {
+    const {name, languages, size} = item;
+    const subscription = name in subscriptions ? subscriptions[name] : null;
+    const subscriptionLanguage = subscription ? subscription['language'] : 'en';
+
+    const [langauge, setLanguage] = useState(subscriptionLanguage);
+    const [pending, setPending] = useState(false);
+    const languageChange = subscription ? langauge !== subscription['language'] : false;
+
+    const handleButton = async (e) => {
+        if (e) {
+            e.preventDefault();
+        }
+        let success = false;
+        try {
+            setPending(true);
+            if (subscription && !languageChange) {
+                success = await zimUnsubscribe(subscription['id']);
+            } else {
+                success = await zimSubscribe(name, langauge);
+            }
+        } catch (e) {
+            console.error(e);
+        }
+        setPending(false);
+        if (!success) {
+            toast({
+                type: 'error',
+                title: 'Error!',
+                description: subscription ? 'Failed to unsubscribe' : 'Failed to subscribe',
+            })
+        }
+        if (fetchSubscriptions) {
+            await fetchSubscriptions();
+        }
+    }
+
+    const handleLanguageChange = (e, {value}) => {
+        if (e) {
+            e.preventDefault();
+        }
+        setLanguage(value);
+    }
+
+    const languageOptions = languages.map(i => {
+        return {key: i, value: i, text: iso_639_codes[i]}
+    });
+    const languageDropdown = <Dropdown fluid search selection
+                                       placeholder='Language'
+                                       options={languageOptions}
+                                       value={langauge}
+                                       onChange={handleLanguageChange}
+    />;
+
+    const subscribeButton = <Button
+        disabled={pending}
+        onClick={handleButton}>
+        {subscription && !languageChange ? 'Unsubscribe' : 'Subscribe'}
+    </Button>
+
+    return <TableRow key={name}>
+        <TableCell>{name}</TableCell>
+        <TableCell>{languageDropdown}</TableCell>
+        <TableCell>{subscribeButton}</TableCell>
+        <TableCell>{humanFileSize(size)}</TableCell>
+    </TableRow>
+}
+
+class ManageZim extends React.Component {
+    constructor(props) {
+        super(props);
+        this.state = {
+            catalog: undefined,
+            iso_639_codes: undefined,
+            subscriptions: undefined,
+            zims: undefined,
+        }
+    }
+
+    async componentDidMount() {
+        await this.fetchZims();
+        await this.fetchSubscriptions();
+    }
+
+    async fetchZims() {
+        try {
+            const {zims} = await fetchZims();
+            this.setState({zims});
+        } catch (e) {
+            console.error(e);
+        }
+    }
+
+    async fetchSubscriptions() {
+        try {
+            const {subscriptions, catalog, iso_639_codes} = await fetchZimSubscriptions();
+            this.setState({subscriptions, catalog, iso_639_codes});
+        } catch (e) {
+            console.error(e);
+        }
+    }
+
+    zimFileTableRow = (zim, sortData) => {
+        const {path, size} = zim;
+
+        return <TableRow key={path}>
+            <TableCell>{path}</TableCell>
+            <TableCell>{humanFileSize(size)}</TableCell>
+        </TableRow>
+    }
+
+    render() {
+        const {zims, catalog, iso_639_codes, subscriptions} = this.state;
+
+        const zimFilesHeaders = [
+            {key: 'path', text: 'Path', sortBy: 'path', width: 14},
+            {key: 'size', text: 'Size', sortBy: 'size', width: 2},
+        ];
+        let zimFilesBody = <Placeholder>
+            <PlaceholderHeader>
+                <PlaceholderLine/>
+                <PlaceholderLine/>
+            </PlaceholderHeader>
+        </Placeholder>;
+        if (zims && zims.length > 0) {
+            zimFilesBody = <SortableTable
+                data={zims}
+                rowFunc={this.zimFileTableRow}
+                rowKey='path'
+                tableHeaders={zimFilesHeaders}
+            />;
+        } else if (zims === null || (zims && zims.length === 0)) {
+            zimFilesBody = <Message icon warning>
+                <Message.Content>
+                    You have not subscribed to any Kiwix projects, or your files have not been refreshed.
+                </Message.Content>
+            </Message>;
+        }
+
+        const kiwixCatalogHeaders = [
+            {
+                key: 'name', text: 'Name', 'sortBy': [i => {
+                    return i['name'].toLowerCase()
+                }], width: 8
+            },
+            {key: 'language', text: 'Language', 'sortBy': null, width: 4},
+            {key: 'subscription', text: 'Subscription', 'sortBy': null, width: 2},
+            {key: 'size', text: 'Maximum Size', sortBy: 'size', width: 2},
+        ];
+        let kiwixCatalog = <Placeholder>
+            <PlaceholderHeader>
+                <PlaceholderLine/>
+                <PlaceholderLine/>
+            </PlaceholderHeader>
+        </Placeholder>;
+        if (subscriptions && Object.keys(subscriptions).length > 0) {
+            kiwixCatalog = <SortableTable
+                defaultSortColumn='name'
+                data={catalog}
+                rowFunc={i => <ZimCatalogItemRow
+                    iso_639_codes={iso_639_codes}
+                    subscriptions={subscriptions}
+                    item={i}
+                    fetchSubscriptions={async () => await this.fetchSubscriptions()}
+                />}
+                rowKey='name'
+                tableHeaders={kiwixCatalogHeaders}
+                footer={<TableFooter>
+                    <TableRow>
+                        <TableHeaderCell colSpan={4}>
+                            <p>More Zim files are available from the full Kiwix library&nbsp;
+                                <a href='https://download.kiwix.org/'>https://download.kiwix.org/</a>
+                            </p>
+                        </TableHeaderCell>
+                    </TableRow>
+                </TableFooter>}
+            />
+        }
+
+        return <PageContainer>
+            <Header as='h2'>Zim Files</Header>
+            {zimFilesBody}
+
+            <Header as='h2'>Kiwix Catalog</Header>
+            {kiwixCatalog}
+
+            <Divider/>
+
+            <ViewerMessage/>
+        </PageContainer>
+    }
+}
+
+function ZimApp() {
+    return <iframe
+        title='zim'
+        src={VIEWER_URL}
+        style={{
+            position: 'fixed',
+            height: '100%',
+            width: '100%',
+            border: 'none',
+            backgroundColor: '#FFFFFF',
+        }}/>
+}
+
+export function ZimRoute() {
+    useTitle('Zim');
+
+    const links = [
+        {text: 'Viewer', to: '/zim', key: 'zim', end: true},
+        {text: 'Manage', to: '/zim/manage', key: 'manage'},
+    ]
+
+    return <div style={{marginTop: '2em'}}>
+        <TabLinks links={links}/>
+        <Routes>
+            <Route path='/' exact element={<ZimApp/>}/>
+            <Route path='manage' exact element={<ManageZim/>}/>
+        </Routes>
+    </div>
+}

app/src/components/admin/Settings.js

@@ -25,6 +25,7 @@ export class Settings extends React.Component {
             hotspot_password: null,
             hotspot_ssid: null,
             hotspot_status: null,
+            ignore_outdated_zims: null,
             throttle_on_startup: null,
             throttle_status: null,
         }
@@ -46,6 +47,7 @@ export class Settings extends React.Component {
                 hotspot_password: settings.hotspot_password,
                 hotspot_ssid: settings.hotspot_ssid,
                 hotspot_status: settings.hotspot_status,
+                ignore_outdated_zims: settings.ignore_outdated_zims,
                 throttle_on_startup: settings.throttle_on_startup,
                 throttle_status: settings.throttle_status,
             }, this.handleHotspotChange);
@@ -64,6 +66,7 @@ export class Settings extends React.Component {
             hotspot_on_startup: this.state.hotspot_on_startup,
             hotspot_password: this.state.hotspot_password,
             hotspot_ssid: this.state.hotspot_ssid,
+            ignore_outdated_zims: this.state.ignore_outdated_zims,
             throttle_on_startup: this.state.throttle_on_startup,
         }
         await saveSettings(settings);
@@ -111,6 +114,7 @@ export class Settings extends React.Component {
             hotspot_on_startup,
             hotspot_password,
             hotspot_ssid,
+            ignore_outdated_zims,
             pending,
             qrCodeValue,
             throttle_on_startup,
@@ -163,6 +167,15 @@ export class Settings extends React.Component {
                             />
                         </div>
 
+                        <div style={{margin: '0.5em'}}>
+                            <Toggle
+                                label='Ignore outdated Zims'
+                                disabled={disabled || ignore_outdated_zims === null}
+                                checked={ignore_outdated_zims === true}
+                                onChange={checked => this.handleInputChange(null, 'ignore_outdated_zims', checked)}
+                            />
+                        </div>
+
                         <FormGroup inline>
                             <FormInput
                                 label={<>

app/src/hooks/customHooks.js

@@ -11,6 +11,7 @@ import {
     getDownloads,
     getFiles,
     getInventory,
+    getOutdatedZims,
     getSettings,
     getStatistics,
     getStatus,
@@ -20,6 +21,8 @@ import {
     searchArchives,
     searchDirectories,
     searchVideos,
+    searchZim,
+    searchZims,
     setHotspot,
     setThrottle,
 } from "../api";
@@ -27,13 +30,18 @@ import {createSearchParams, useSearchParams} from "react-router-dom";
 import {enumerate, filterToMimetypes, humanFileSize, secondsToFullDuration} from "../components/Common";
 import {StatusContext} from "../contexts/contexts";
 import {toast} from "react-semantic-toasts-2";
+import {useSearch} from "../components/Search";
 
 const calculatePage = (offset, limit) => {
     return offset && limit ? Math.round((offset / limit) + 1) : 1;
 }
 
 const calculateTotalPages = (total, limit) => {
-    return total && limit ? Math.round(total / limit) : 1;
+    // Return the total divided by the limit, but always at least 1.
+    if (!total || !limit || total < limit) {
+        return 1;
+    }
+    return Math.round(total / limit) + 1;
 }
 
 export const useRecurringTimeout = (callback, delay) => {
@@ -147,11 +155,12 @@ export const useArchive = (archiveId) => {
     return {archiveFile: archiveFileGroup, history, fetchArchive};
 }
 
-export const usePages = (defaultLimit) => {
+export const usePages = (defaultLimit = 24, totalPages = 0) => {
     const {searchParams, updateQuery} = useQuery();
     const offset = searchParams.get('o') || 0;
     const limit = parseInt(searchParams.get('l') || defaultLimit || 24);
     const [activePage, setActivePage] = useState(calculatePage(offset, limit));
+    const [totalPages_, setTotalPages] = useState(totalPages);
 
     const setLimit = (value) => {
         setPage(1);
@@ -159,12 +168,22 @@ export const usePages = (defaultLimit) => {
     }
 
     const setPage = (value) => {
+        console.debug(`setPage ${value}`);
         setActivePage(value);
         value = value - 1;  // Page really starts as 0.
         updateQuery({o: value * limit});
     }
 
-    return {offset, limit, setLimit, activePage, setPage};
+    const setTotal = (total) => {
+        const newTotalPages = calculateTotalPages(total, limit);
+        console.log('newTotalPages', newTotalPages);
+        setTotalPages(newTotalPages);
+    }
+
+    // Used for useEffect.
+    const effect = JSON.stringify({offset, limit, activePage});
+
+    return {offset, limit, setLimit, activePage, setPage, totalPages: totalPages_, setTotal, effect};
 }
 
 export const useSearchArchives = (defaultLimit) => {
@@ -403,18 +422,17 @@ export const useChannels = () => {
 }
 
 export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) => {
-    const {searchParams, updateQuery} = useQuery();
-    const limit = searchParams.get('l') || defaultLimit;
-    const offset = searchParams.get('o') || 0;
-    const searchStr = searchParams.get('q');
-    const filter = searchParams.get('filter');
-    const model_ = searchParams.get('model');
-    const activeTags = searchParams.getAll('tag');
+    const {
+        activeTags,
+        pages,
+        searchStr,
+        filter,
+        model: model_,
+        setSearchStr
+    } = useSearch(defaultLimit, emptySearch, model);
     const {view} = useSearchView();
 
     const [searchFiles, setSearchFiles] = useState(null);
-    const [totalPages, setTotalPages] = useState(0);
-    const [activePage, setActivePage] = useState(calculatePage(offset, limit));
     const headline = view === 'headline';
 
     const localSearchFiles = async () => {
@@ -423,13 +441,13 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
         }
         const mimetypes = filterToMimetypes(filter);
         setSearchFiles(null);
-        setTotalPages(0);
         try {
             let [file_groups, total] = await filesSearch(
-                offset, limit, searchStr, mimetypes, model || model_, activeTags, headline);
+                pages.offset, pages.limit, searchStr, mimetypes, model || model_, activeTags, headline);
             setSearchFiles(file_groups);
-            setTotalPages(calculateTotalPages(total, limit));
+            pages.setTotal(total);
         } catch (e) {
+            pages.setTotal(0);
             console.error(e);
             toast({
                 type: 'error',
@@ -442,25 +460,16 @@ export const useSearchFiles = (defaultLimit = 48, emptySearch = false, model) =>
 
     useEffect(() => {
         localSearchFiles();
-    }, [searchStr, limit, offset, activePage, filter, model, model_, JSON.stringify(activeTags), headline]);
+    }, [searchStr, pages.effect, filter, model, model_, JSON.stringify(activeTags), headline]);
 
-    const setPage = (i) => {
-        i = parseInt(i);
-        let l = parseInt(limit);
-        updateQuery({o: (l * i) - l})
-        setActivePage(i);
-    }
-
-    const setSearchStr = (value) => {
-        updateQuery({q: value, o: null});
-    }
-
-    const setLimit = (value) => {
-        setActivePage(1);
-        updateQuery({l: value, o: 0});
-    }
-
-    return {searchFiles, totalPages, limit, searchStr, filter, setSearchStr, activePage, setPage, setLimit, activeTags};
+    return {
+        searchFiles,
+        searchStr,
+        filter,
+        setSearchStr,
+        pages,
+        activeTags
+    };
 }
 
 export const useBrowseFiles = () => {
@@ -930,6 +939,106 @@ export const useUploadFile = () => {
     return {files, setFiles: handleFilesChange, progresses, destination, setDestination, doClear, doUpload}
 }
 
+export const useSearchZims = (defaultLimit) => {
+    const {offset, limit, setLimit, activePage, setPage} = usePages(defaultLimit);
+    const {searchParams, updateQuery} = useQuery();
+    const searchStr = searchParams.get('q') || '';
+
+    const [zims, setZims] = useState(null);
+    const [totalPages, setTotalPages] = useState(0);
+
+    const localSearchZims = async () => {
+        setZims(null);
+        try {
+            let [zims_,] = await searchZims(offset, limit, searchStr);
+            setZims(zims_);
+        } catch (e) {
+            console.error(e);
+            toast({
+                type: 'error',
+                title: 'Unexpected server response',
+                description: 'Could not get archives',
+                time: 5000,
+            });
+            setZims([]);
+        }
+    }
+
+    useEffect(() => {
+        localSearchZims();
+    }, [searchStr, limit, activePage, setZims]);
+
+    const setSearchStr = (value) => {
+        updateQuery({q: value, o: 0, order: undefined});
+    }
+
+    const setOrderBy = (value) => {
+        setPage(1);
+        updateQuery({order: value});
+    }
+
+    return {
+        zims,
+        limit,
+        setLimit,
+        offset,
+        setOrderBy,
+        totalPages,
+        activePage,
+        setPage,
+        searchStr,
+        setSearchStr,
+        fetchArchives: localSearchZims,
+    }
+}
+
+
+export const useSearchZim = (searchStr, zimId, active, activeTags, defaultLimit = 10) => {
+    const [zim, setZim] = useState(null);
+    const pages = usePages(defaultLimit);
+    const [loading, setLoading] = useState(false);
+
+    const localFetchSearch = async () => {
+        if (!active) {
+            return;
+        }
+        setLoading(true);
+        try {
+            const zim = await searchZim(pages.offset, pages.limit, searchStr, zimId, activeTags);
+            setZim(zim);
+            pages.setTotal(zim.estimate);
+        } catch (e) {
+            pages.setTotal(0);
+            console.error(`Failed to search Zim ${zimId}`);
+            console.error(e);
+        }
+        setLoading(false);
+    }
+
+    useEffect(() => {
+        localFetchSearch();
+    }, [active, searchStr, JSON.stringify(activeTags), pages.effect]);
+
+    return {zim, fetchSearch: localFetchSearch, pages, loading}
+}
+
+export const useOutdatedZims = () => {
+    const [outdated, setOutdated] = useState(null);
+    const [current, setCurrent] = useState(null);
+
+    const localFetchOutdatedZims = async () => {
+        const zims = await getOutdatedZims();
+        setOutdated(zims['outdated']);
+        setCurrent(zims['current']);
+    }
+
+    useEffect(() => {
+        localFetchOutdatedZims();
+    }, []);
+
+    return {outdated, current}
+}
+
 export const useVINDecoder = (defaultVINNumber = '') => {
     const [value, setValue] = useState(defaultVINNumber);
     // The response vin from the API.

conftest.py

@@ -1,6 +1,5 @@
-# Import conftest for test fixtures
-from wrolpi.common import import_modules
+from modules.archive.conftest import *  # noqa
+from modules.inventory.conftest import *  # noqa
+from modules.videos.conftest import *  # noqa
+from modules.zim.conftest import *  # noqa
 from wrolpi.conftest import *  # noqa
-
-# Import modules and their conftest.py.
-import_modules()

docker-compose.yml

@@ -92,6 +92,15 @@ services:
     healthcheck:
       test: [ 'CMD-SHELL', 'curl http://127.0.0.1:80' ]
 
+  zim:
+    build:
+      context: .
+      dockerfile: docker/zim/Dockerfile
+    volumes:
+      - './test:/media/wrolpi'
+    ports:
+      - '8085:8085'
+
 volumes:
   openstreetmap-data:
     external: true

docker/zim/Dockerfile

@@ -0,0 +1,7 @@
+FROM debian:12
+ENV DOCKER=true
+
+RUN apt update && apt-get install -y kiwix kiwix-tools
+COPY docker/zim/* /
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/zim/entrypoint.sh

@@ -0,0 +1,7 @@
+#! /usr/bin/env bash
+rm -f /library.xml
+
+# Find all Zim files and import them.
+find /media/wrolpi -iname '*.zim' -exec /import_zim.sh {} \;
+
+kiwix-serve --library /library.xml --address 0.0.0.0 --port 8085

docker/zim/import_zim.sh

@@ -0,0 +1,7 @@
+#! /usr/bin/env bash
+# This should not be used outside the Zim docker container!!
+
+# This file echos and imports Zim files.
+
+echo Importing zim $*
+kiwix-manage /library.xml add $*

main.py

@@ -9,7 +9,7 @@ from sanic.signals import Event
 
 from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS
 from wrolpi import root_api, admin
-from wrolpi.common import logger, get_config, import_modules, check_media_directory, limit_concurrent, \
+from wrolpi.common import logger, get_config, check_media_directory, limit_concurrent, \
     wrol_mode_enabled, cancel_refresh_tasks, set_log_level
 from wrolpi.downloader import download_manager, import_downloads_config
 from wrolpi.root_api import api_app
@@ -66,7 +66,6 @@ def launch_interactive_shell():
     import code
     from wrolpi.db import get_db_session
 
-    modules = import_modules()
     with get_db_session() as session:
         code.interact(banner=INTERACTIVE_BANNER, local=locals())
 
@@ -147,8 +146,8 @@ def main():
 
     check_media_directory()
 
-    # Import the API in every module.  Each API should attach itself to `root_api`.
-    import_modules()
+    # Import modules before calling BEFORE_STARTUP_FUNCTIONS.
+    import modules  # noqa
 
     # Run the startup functions
     for func in BEFORE_STARTUP_FUNCTIONS:
@@ -172,6 +171,9 @@ async def startup(app: Sanic):
     flags.init_flags()
     await import_downloads_config()
 
+    from modules.zim.lib import flag_outdated_zim_files
+    flag_outdated_zim_files()
+
 
 @api_app.after_server_start
 @limit_concurrent(1)

modules/__init__.py

@@ -0,0 +1,6 @@
+from .archive import api
+from .inventory import api
+from .map import api
+from .otp import api
+from .videos import api
+from .zim import api, test

modules/archive/__init__.py

@@ -1,7 +1,7 @@
 import asyncio
 import json
 from abc import ABC
-from typing import Tuple, List
+from typing import List
 
 from sqlalchemy import not_
 from sqlalchemy.orm import Session

modules/archive/lib.py

@@ -20,7 +20,8 @@ from wrolpi.common import get_media_directory, logger, extract_domain, chdir, es
 from wrolpi.dates import now, Seconds
 from wrolpi.db import get_db_session, get_db_curs, optional_session
 from wrolpi.errors import UnknownArchive, InvalidOrderBy
-from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_sub_select
+from wrolpi.files.lib import handle_file_group_search_results
+from wrolpi.tags import tag_names_to_file_group_sub_select
 from wrolpi.vars import DOCKERIZED, PYTEST
 
 logger = logger.getChild(__name__)
@@ -417,7 +418,7 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
 
     if tag_names:
         # Filter all FileGroups by those that have been tagged with the provided tag names.
-        tags_stmt, params_ = tag_names_to_sub_select(tag_names)
+        tags_stmt, params_ = tag_names_to_file_group_sub_select(tag_names)
         params.update(params_)
         wheres.append(f'fg.id = ANY({tags_stmt})')
 

modules/inventory/schema.py

@@ -3,8 +3,6 @@ from datetime import datetime
 from decimal import Decimal
 from typing import Union, Optional
 
-from wrolpi.dates import from_timestamp
-
 
 @dataclass
 class ItemPostRequest:

modules/inventory/test/test_api.py

@@ -1,6 +1,8 @@
 import json
 from http import HTTPStatus
 
+import pytest
+
 from modules.inventory import Item
 from modules.inventory.inventory import save_item
 
@@ -73,33 +75,34 @@ def test_item_api(test_session, init_test_inventory, test_client):
         assert response.status_code == HTTPStatus.NO_CONTENT, response.json
 
 
-def test_inventory_api(test_session, test_client):
+@pytest.mark.asyncio
+async def test_inventory_api(test_session, test_async_client):
     """An Inventory can be created."""
     inventory = dict(name='foo')
-    request, response = test_client.post('/api/inventory', content=json.dumps(inventory))
+    request, response = await test_async_client.post('/api/inventory', content=json.dumps(inventory))
     assert response.status_code == HTTPStatus.CREATED, response.status_code
 
-    request, response = test_client.get('/api/inventory')
+    request, response = await test_async_client.get('/api/inventory')
     assert response.status_code == HTTPStatus.OK
     inventory = response.json['inventories'][0]
     inventory_id = inventory['id']
 
-    request, response = test_client.get(f'/api/inventory/{inventory_id}')
+    request, response = await test_async_client.get(f'/api/inventory/{inventory_id}')
     assert response.status_code == HTTPStatus.OK
 
-    request, response = test_client.get('/api/inventory/categories')
+    request, response = await test_async_client.get('/api/inventory/categories')
     assert response.status_code == HTTPStatus.OK
 
-    request, response = test_client.get('/api/inventory/brands')
+    request, response = await test_async_client.get('/api/inventory/brands')
     assert response.status_code == HTTPStatus.OK
 
     inventory = dict(name='new name')
-    request, response = test_client.put(f'/api/inventory/{inventory_id}', content=json.dumps(inventory))
+    request, response = await test_async_client.put(f'/api/inventory/{inventory_id}', content=json.dumps(inventory))
     assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
 
-    request, response = test_client.delete(f'/api/inventory/{inventory_id}')
+    request, response = await test_async_client.delete(f'/api/inventory/{inventory_id}')
     assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
 
-    request, response = test_client.get('/api/inventory')
+    request, response = await test_async_client.get('/api/inventory')
     assert response.status_code == HTTPStatus.OK
     assert len(response.json['inventories']) == 0

modules/videos/channel/lib.py

@@ -1,4 +1,3 @@
-import pathlib
 from pathlib import Path
 from typing import List, Dict, Union
 

modules/videos/channel/test/test_api.py

@@ -332,7 +332,8 @@ def test_channel_by_id(test_session, test_client, simple_channel, simple_video):
     assert response.status_code == HTTPStatus.OK
 
 
-def test_channel_crud(test_session, test_client, test_directory, test_download_manager):
+@pytest.mark.asyncio
+async def test_channel_crud(test_session, test_async_client, test_directory, test_download_manager):
     channel_directory = test_directory / 'channel directory'
     channel_directory.mkdir()
 
@@ -344,15 +345,15 @@ def test_channel_crud(test_session, test_client, test_directory, test_download_m
     )
 
     # Channel doesn't exist
-    request, response = test_client.get('/api/videos/channels/examplechannel1')
+    request, response = await test_async_client.get('/api/videos/channels/examplechannel1')
     assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
 
     # Create it
-    request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
+    request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(new_channel))
     assert response.status_code == HTTPStatus.CREATED, response.json
     location = response.headers['Location']
 
-    request, response = test_client.get(location)
+    request, response = await test_async_client.get(location)
     assert response.status_code == HTTPStatus.OK, response.json
     created = response.json['channel']
     assert created
@@ -369,13 +370,13 @@ def test_channel_crud(test_session, test_client, test_directory, test_download_m
         f'Channel directory is absolute: {created["directory"]}'
 
     # Can't create it again
-    request, response = test_client.post('/api/videos/channels', content=json.dumps(new_channel))
+    request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(new_channel))
     assert response.status_code == HTTPStatus.BAD_REQUEST
 
-    def put_and_fetch(new_channel_):
-        request, response = test_client.put(location, content=json.dumps(new_channel))
+    async def put_and_fetch(new_channel_):
+        request, response = await test_async_client.put(location, content=json.dumps(new_channel))
         assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
-        request, response = test_client.get(location)
+        request, response = await test_async_client.get(location)
         assert response.status_code == HTTPStatus.OK
         channel = response.json['channel']
         return channel
@@ -384,7 +385,7 @@ def test_channel_crud(test_session, test_client, test_directory, test_download_m
     new_channel['name'] = 'Example Channel 2'
     new_channel['directory'] = str(new_channel['directory'])  # noqa
     new_channel['download_frequency'] = 60
-    channel = put_and_fetch(new_channel)
+    channel = await put_and_fetch(new_channel)
     assert channel['id'] == 1
     assert channel['name'] == 'Example Channel 2'
     assert channel['directory'] == channel_directory.name
@@ -397,22 +398,22 @@ def test_channel_crud(test_session, test_client, test_directory, test_download_m
 
     # Update with no download frequency.
     new_channel['download_frequency'] = None
-    channel = put_and_fetch(new_channel)
+    channel = await put_and_fetch(new_channel)
     assert channel['download_frequency'] is None
     # Download was deleted.
     assert len(test_download_manager.get_downloads(test_session)) == 0
 
     # Can't update channel that doesn't exist
-    request, response = test_client.put('/api/videos/channels/doesnt_exist',
+    request, response = await test_async_client.put('/api/videos/channels/doesnt_exist',
                                                     content=json.dumps(new_channel))
     assert response.status_code == HTTPStatus.NOT_FOUND
 
     # Delete the new channel
-    request, response = test_client.delete(location)
+    request, response = await test_async_client.delete(location)
     assert response.status_code == HTTPStatus.NO_CONTENT
 
     # Cant delete it again
-    request, response = test_client.delete(location)
+    request, response = await test_async_client.delete(location)
     assert response.status_code == HTTPStatus.NOT_FOUND
 
 
@@ -433,7 +434,8 @@ def test_channel_creation_with_download(test_client, test_session, test_director
     assert download.sub_downloader == 'video'
 
 
-def test_change_channel_url(test_client, test_session, test_download_manager, download_channel,
+@pytest.mark.asyncio
+async def test_change_channel_url(test_async_client, test_session, test_download_manager, download_channel,
                                   test_download_manager_config):
     # Change the Channel's URL.
     download_channel.update({'url': 'https://example.com/new-url'})

modules/videos/channel/test/test_lib.py

@@ -1,14 +1,10 @@
-import asyncio
-from unittest import mock
-
 import pytest
 
 from modules.videos import schema
 from modules.videos.channel import lib
 from modules.videos.lib import save_channels_config, import_channels_config
-from modules.videos.models import Channel, Video
+from modules.videos.models import Channel
 from wrolpi.errors import UnknownChannel
-from wrolpi.files.lib import refresh_files
 
 
 @pytest.mark.parametrize('params', [

modules/videos/channel/test/test_models.py

@@ -1,8 +1,3 @@
-import pytest
-
-from modules.videos import Channel
-
-
 def test_delete_channel_no_url(test_session, test_client, channel_factory):
     """
     A Channel can be deleted even if it has no URL.

modules/videos/downloader.py

@@ -4,14 +4,13 @@ import pathlib
 import re
 import traceback
 from abc import ABC
-from typing import Tuple, List, Optional, Dict
+from typing import Tuple, List, Dict
 from urllib.parse import urlparse
 
 import yt_dlp.utils
 from sqlalchemy.orm import Session
 from yt_dlp import YoutubeDL
 from yt_dlp.extractor import YoutubeTabIE  # noqa
-from yt_dlp.utils import UnsupportedError, DownloadError
 
 from wrolpi.cmd import YT_DLP_BIN
 from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task

modules/videos/lib.py

@@ -1,5 +1,4 @@
 import contextlib
-import datetime
 import functools
 import html
 import pathlib

modules/videos/models.py

@@ -4,7 +4,7 @@ import pathlib
 from pathlib import Path
 from typing import Optional, Dict, List, Union
 
-from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, BigInteger, any_
+from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, ForeignKey, BigInteger
 from sqlalchemy.orm import relationship, Session, deferred
 from sqlalchemy.orm.collections import InstrumentedList
 

modules/videos/schema.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, field
-from datetime import date, datetime
+from datetime import date
 from typing import List, Optional
 
 from modules.videos.video.lib import DEFAULT_VIDEO_ORDER, VIDEO_QUERY_LIMIT

modules/videos/test/test_api.py

@@ -2,14 +2,13 @@ import json
 from http import HTTPStatus
 from pathlib import Path
 
-import mock.mock
 import pytest
 
 from modules.videos.models import Video, Channel
 from modules.videos.video import lib as video_lib
 from wrolpi.db import get_db_curs
 from wrolpi.downloader import DownloadFrequency, Download
-from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
+from wrolpi.files.lib import refresh_files
 from wrolpi.files.models import FileGroup
 
 
@@ -53,12 +52,12 @@ def test_refresh_videos(test_client, test_session, test_directory, simple_channe
                            with_info_json=True, with_poster_ext='jpg')
     test_session.commit()
     video1.file_group.size = video1.file_group.data = None
-    video1.file_group.files = list()
+    video1.file_group.paths = list()
     # video2 is in the test directory.
     video2 = video_factory(channel_id=simple_channel.id, with_video_file=True, with_info_json=True,
                            with_poster_ext='jpg')
     video2.file_group.data = None
-    video2.file_group.files = []
+    video2.file_group.paths = []
     test_session.commit()
 
     assert not video1.file_group.size, 'video1 should not have size during creation'

modules/videos/test/test_common.py

@@ -1,17 +1,13 @@
 import pathlib
 import subprocess
 import tempfile
-from datetime import datetime
-from typing import List
 from unittest import mock
 
 import pytest
-import pytz
 from PIL import Image
 
 from modules.videos.models import Channel, Video
-from wrolpi.common import get_absolute_media_path, sanitize_link
-from wrolpi.dates import  now
+from wrolpi.common import get_absolute_media_path
 from wrolpi.downloader import Download, DownloadFrequency
 from wrolpi.vars import PROJECT_DIR
 from .. import common

modules/videos/test/test_downloader.py

@@ -158,6 +158,7 @@ async def test_download_video_tags(test_session, video_download_manager, video_f
 
 
 @skip_circleci
+@pytest.mark.skip  # TODO this test fails when running multiple tests.
 @pytest.mark.asyncio
 async def test_download_channel(test_session, simple_channel, video_download_manager, video_file,
                                 mock_video_extract_info, mock_video_prepare_filename,

modules/videos/test/test_video.py

@@ -1,7 +1,6 @@
 import pytest
 
 from modules.videos.models import Video
-from wrolpi.dates import now
 from wrolpi.errors import FileGroupIsTagged
 from wrolpi.files import lib as files_lib
 from wrolpi.files.models import FileGroup

modules/videos/video/lib.py

@@ -7,7 +7,7 @@ from modules.videos.models import Video
 from wrolpi.common import run_after, logger
 from wrolpi.db import get_db_session, optional_session
 from wrolpi.errors import UnknownVideo
-from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_sub_select
+from wrolpi.files.lib import handle_file_group_search_results, tag_names_to_file_group_sub_select
 from ..lib import save_channels_config
 
 logger.getChild(__name__)
@@ -103,7 +103,7 @@ def search_videos(
 
     if tag_names:
         # Filter all FileGroups by those that have been tagged with the provided tag names.
-        tags_stmt, params_ = tag_names_to_sub_select(tag_names)
+        tags_stmt, params_ = tag_names_to_file_group_sub_select(tag_names)
         params.update(params_)
         wheres.append(f'fg.id = ANY({tags_stmt})')
 

modules/videos/video/test/test_api.py

@@ -9,7 +9,6 @@ from modules.videos.models import Video
 from modules.videos.video.lib import get_video_for_app
 from wrolpi.dates import now
 from wrolpi.errors import API_ERRORS, WROLModeEnabled
-from wrolpi.root_api import api_app
 
 
 def test_get_video_prev_next(test_session, channel_factory, video_factory):
@@ -121,7 +120,10 @@ def test_video_delete(test_client, test_session, test_directory, channel_factory
     assert response.status_code == HTTPStatus.BAD_REQUEST
 
 
-def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixture, test_download_manager, tag_factory):
+@pytest.mark.asyncio
+async def test_wrol_mode(test_async_client, test_directory, simple_channel, simple_video, wrol_mode_fixture,
+                         test_download_manager,
+                         tag_factory):
     """Many methods are blocked when WROL Mode is enabled."""
     channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory)))
     tag = tag_factory()
@@ -129,32 +131,32 @@ def test_wrol_mode(test_directory, simple_channel, simple_video, wrol_mode_fixtu
     wrol_mode_fixture(True)
 
     # Can't create, update, or delete a channel.
-    _, resp = api_app.test_client.post('/api/videos/channels', content=channel)
+    _, resp = await test_async_client.post('/api/videos/channels', content=channel)
     assert resp.status_code == HTTPStatus.FORBIDDEN
     assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
-    _, resp = api_app.test_client.put(f'/api/videos/channels/{simple_channel.id}', content=channel)
+    _, resp = await test_async_client.put(f'/api/videos/channels/{simple_channel.id}', content=channel)
     assert resp.status_code == HTTPStatus.FORBIDDEN
     assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
-    _, resp = api_app.test_client.delete(f'/api/videos/channels/{simple_channel.id}')
+    _, resp = await test_async_client.delete(f'/api/videos/channels/{simple_channel.id}')
     assert resp.status_code == HTTPStatus.FORBIDDEN
     assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
 
     # Can't delete a video
-    _, resp = api_app.test_client.delete(f'/api/videos/video/{simple_video.id}')
+    _, resp = await test_async_client.delete(f'/api/videos/video/{simple_video.id}')
     assert resp.status_code == HTTPStatus.FORBIDDEN
     assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
 
     # Can't refresh or download
-    _, resp = api_app.test_client.post('/api/files/refresh')
+    _, resp = await test_async_client.post('/api/files/refresh')
     assert resp.status_code == HTTPStatus.FORBIDDEN
     assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
-    _, resp = api_app.test_client.post(f'/api/videos/channels/download/{simple_channel.id}')
+    _, resp = await test_async_client.post(f'/api/videos/channels/download/{simple_channel.id}')
     assert resp.status_code == HTTPStatus.FORBIDDEN
     assert resp.json['code'] == API_ERRORS[WROLModeEnabled]['code']
 
     # THE REST OF THESE METHODS ARE ALLOWED
     content = dict(video_id=simple_video.id, tag_name=tag.name)
-    _, resp = api_app.test_client.post('/api/videos/tag', content=json.dumps(content))
+    _, resp = await test_async_client.post('/api/videos/tag', content=json.dumps(content))
     assert resp.status_code == HTTPStatus.NO_CONTENT
 
     assert test_download_manager.stopped.is_set()

modules/zim/__init__.py

@@ -0,0 +1,3 @@
+from .downloader import KiwixCatalogDownloader  # noqa
+
+__all__ = []

modules/zim/api.py

@@ -0,0 +1,156 @@
+import urllib.parse
+from http import HTTPStatus
+
+from sanic import Request, response
+from sanic_ext import validate
+from sanic_ext.extensions.openapi import openapi
+
+from wrolpi import lang
+from wrolpi.common import logger
+from wrolpi.db import get_db_session
+from wrolpi.downloader import download_manager
+from wrolpi.events import Events
+from wrolpi.root_api import get_blueprint, json_response
+from . import lib, schema
+
+bp = get_blueprint('Zim', '/api/zim')
+
+logger = logger.getChild(__name__)
+
+
+@bp.get('/')
+@openapi.definition(
+    summary='List all known Zim files',
+    response=schema.GetZimsResponse,
+)
+async def get_zims(_: Request):
+    with get_db_session() as session:
+        zims = lib.get_zims(session=session)
+        resp = {
+            'zims': [i.__json__() for i in zims],
+        }
+    return json_response(resp)
+
+
+@bp.delete('/<zim_ids:[0-9,]+>')
+@openapi.definition(
+    summary='Delete all Zim files',
+)
+async def delete_zims(_: Request, zim_ids: str):
+    zim_ids = [int(i) for i in zim_ids.split(',')]
+    lib.delete_zims(zim_ids)
+    return response.empty()
+
+
+@bp.post('/search/<zim_id:[0-9]+>')
+@openapi.definition(
+    summary='Search all entries of a Zim',
+    body=schema.ZimSearchRequest,
+    response=schema.ZimSearchResponse,
+)
+@validate(schema.ZimSearchRequest)
+async def search_zim(_: Request, zim_id: int, body: schema.ZimSearchRequest):
+    headlines = lib.headline_zim(body.search_str, zim_id, tag_names=body.tag_names, offset=body.offset,
+                                 limit=body.limit)
+    return json_response({'zim': headlines})
+
+
+@bp.get('/<zim_id:[0-9]+>/entry/<zim_path:[ -~/]*>')
+@openapi.definition(
+    summary='Read the entry at `zim_path` from the Zim file',
+)
+async def get_zim_entry(_: Request, zim_id: int, zim_path: str):
+    zim_path = urllib.parse.unquote(zim_path)
+    entry = lib.get_entry(zim_path, zim_id=zim_id)
+    content = bytes(entry.get_item().content)
+    try:
+        resp = response.html(content.decode('UTF-8'))
+        logger.debug(f'Returning ZIM HTML response {repr(str(zim_path))}')
+    except UnicodeDecodeError:
+        resp = response.raw(content)
+        logger.debug(f'Returning ZIM raw response {repr(str(zim_path))}')
+
+    return resp
+
+
+@bp.post('/tag')
+@openapi.definition(
+    summary='Tag a Zim entry',
+    body=schema.TagZimEntry,
+)
+@validate(schema.TagZimEntry)
+async def post_zim_tag(_: Request, body: schema.TagZimEntry):
+    await lib.add_tag(body.tag_name, body.zim_id, body.zim_entry)
+    return response.empty(HTTPStatus.CREATED)
+
+
+@bp.post('/untag')
+@openapi.definition(
+    summary='Untag a Zim entry',
+    body=schema.TagZimEntry,
+)
+@validate(schema.TagZimEntry)
+async def post_zim_untag(_: Request, body: schema.TagZimEntry):
+    await lib.untag(body.tag_name, body.zim_id, body.zim_entry)
+    return response.empty(HTTPStatus.NO_CONTENT)
+
+
+@bp.get('/subscribe')
+@openapi.definition(
+    summary='Retrieve Zim subscriptions',
+    response=schema.ZimSubscriptions,
+)
+async def get_zim_subscriptions(_: Request):
+    catalog = lib.get_kiwix_catalog()
+    with get_db_session() as session:
+        subscriptions = {i: j.__json__() for i, j in lib.get_kiwix_subscriptions(session).items()}
+    resp = dict(
+        catalog=catalog,
+        iso_639_codes=lang.ISO_639_CODES,
+        subscriptions=subscriptions,
+    )
+    return json_response(resp)
+
+
+@bp.post('/subscribe')
+@openapi.definition(
+    summary='Subscribe to a particular Kiwix Zim',
+    body=schema.TagZimSubscribe,
+)
+@validate(schema.TagZimSubscribe)
+async def post_zim_subscribe(_: Request, body: schema.TagZimSubscribe):
+    await lib.subscribe(body.name, body.language)
+    if download_manager.disabled.is_set() or download_manager.stopped.is_set():
+        # Downloads are disabled, warn the user.
+        Events.send_downloads_disabled('Download created. But, downloads are disabled.')
+    return response.empty(HTTPStatus.CREATED)
+
+
+@bp.delete('/subscribe/<subscription_id:[0-9]+>')
+@openapi.definition(
+    summary='Unsubscribe to a particular Kiwix Zim',
+)
+async def delete_zim_subscription(_: Request, subscription_id: int):
+    await lib.unsubscribe(subscription_id)
+    return response.empty(HTTPStatus.NO_CONTENT)
+
+
+@bp.get('/outdated')
+@openapi.definition(
+    summary='Returns the outdated and current Zim files',
+    response=schema.OutdatedZims,
+)
+async def get_outdated_zims(_: Request):
+    outdated, current = lib.find_outdated_zim_files()
+    d = dict(outdated=outdated, current=current)
+    return json_response(d)
+
+
+@bp.delete('/outdated')
+@openapi.definition(
+    summary='Remove all outdated Zims, if any.'
+)
+async def delete_outdated_zims(_: Request):
+    await lib.remove_outdated_zim_files()
+    lib.flag_outdated_zim_files()
+    return response.empty(HTTPStatus.NO_CONTENT)

modules/zim/conftest.py

@@ -0,0 +1,130 @@
+import asyncio
+import json
+import pathlib
+from http import HTTPStatus
+from itertools import zip_longest
+from typing import List, Dict
+from uuid import uuid4
+
+import pytest
+from mock import mock
+
+from modules.zim.downloader import KiwixCatalogDownloader, KiwixZimDownloader
+from modules.zim.models import Zim
+from wrolpi.common import DownloadFileInfo
+from wrolpi.downloader import DownloadManager
+from wrolpi.vars import PROJECT_DIR
+
+
+@pytest.fixture
+def kiwix_download_manager(test_download_manager) -> DownloadManager:
+    kiwix_downloader = KiwixCatalogDownloader()
+    kiwix_zim_downloader = KiwixZimDownloader()
+
+    test_download_manager.register_downloader(kiwix_downloader)
+    test_download_manager.register_downloader(kiwix_zim_downloader)
+
+    yield test_download_manager
+
+
+@pytest.fixture
+def kiwix_download_zim(test_directory, kiwix_download_manager, test_zim_bytes):
+    async def do_download(download_info: DownloadFileInfo = None, expected_url: str = None,
+                          download_file_side_effect=None, hrefs: List[str] = None):
+        async def _download_file(url: str, output_path: pathlib.Path = None, info: DownloadFileInfo = None):
+            if expected_url:
+                assert url == expected_url
+            if download_info:
+                assert info == download_info
+            output_path.write_bytes(test_zim_bytes)
+
+        with mock.patch('modules.zim.downloader.fetch_hrefs') as mock_fetch_hrefs, \
+                mock.patch('modules.zim.downloader.get_download_info') as mock_get_download_info, \
+                mock.patch('modules.zim.downloader.download_file') as mock_download_file:
+            # Fake the download, but make sure it is called correctly.  Also creates a Zim file at the output_path.
+            mock_download_file.side_effect = download_file_side_effect or _download_file
+            # These are the links in the parent directory of the url.
+            mock_fetch_hrefs.return_value = hrefs or ['?C=N;O=D', '?C=M;O=A', '?C=S;O=A', '?C=D;O=A', '/zim/',
+                                                      'wikipedia_es_all_maxi_2023-05.zim',
+                                                      'wikipedia_es_all_maxi_2023-06.zim',
+                                                      'wikipedia_es_all_nopic_2023-05.zim',
+                                                      'wikipedia_es_all_nopic_2023-06.zim']
+            # This is the DownloadInfo for the latest Zim file that was selected.
+            mock_get_download_info.return_value = download_info or DownloadFileInfo(
+                status=HTTPStatus.OK,
+                name='wikipedia_es_all_maxi_2023-06.zim',
+                size=len(test_zim_bytes),
+            )
+
+            await kiwix_download_manager.wait_for_all_downloads()
+            # Async download are hard to test, sleep to let tasks finish.
+            await asyncio.sleep(0.1)
+            await kiwix_download_manager.wait_for_all_downloads()
+
+    return do_download
+
+
+@pytest.fixture()
+def test_zim_bytes():
+    """Read the byte contents of the test Zim (test/zim.zim)."""
+    path = PROJECT_DIR / 'test/zim.zim'
+    return path.read_bytes()
+
+
+@pytest.fixture
+def zim_path_factory(test_zim_bytes, test_directory):
+    """Copy the test Zim into the test directory."""
+
+    def _(name: str = None) -> pathlib.Path:
+        name = name if name else f'{uuid4()}.zim'
+        path = test_directory / name
+        path.write_bytes(test_zim_bytes)
+        return path
+
+    return _
+
+
+@pytest.fixture
+def test_zim(test_session, zim_path_factory, test_directory) -> Zim:
+    zim = Zim.from_paths(test_session, zim_path_factory())
+    return zim
+
+
+@pytest.fixture
+def zim_factory(test_session, zim_path_factory, test_directory):
+    def _(name: str) -> Zim:
+        zim = Zim.from_paths(test_session, zim_path_factory(name))
+        return zim
+
+    return _
+
+
+@pytest.fixture
+def assert_zim_search(test_client, test_directory):
+    from http import HTTPStatus
+
+    def _(search_str: str, zim_id: int, expected: Dict, tag_names: List[str] = [],
+          expected_status_code: int = HTTPStatus.OK):
+        content = dict(search_str=search_str, tag_names=tag_names)
+        assert isinstance(zim_id, int), 'This fixture only supports Zim ids of type integer'
+        request, response = test_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content))
+        assert response.status_code == expected_status_code
+
+        result = response.json['zim']
+
+        if path := expected.get('path'):
+            assert result['path'] == str(path.relative_to(test_directory))
+
+        if estimate := expected.get('estimate'):
+            assert result['estimate'] == estimate
+
+        for response_entry, expected_entry in zip_longest(result['search'], expected['search']):
+            if response_entry and not expected_entry:
+                raise AssertionError(f'Response contained more entries than expected: {response_entry}')
+            elif not response_entry and expected_entry:
+                raise AssertionError(f'Response did not contain the expected entry: {expected_entry}')
+            assert response_entry['path'] == expected_entry['path']
+            assert response_entry['headline'] == expected_entry['headline']
+            assert response_entry['rank'] == expected_entry['rank']
+
+    return _

modules/zim/downloader.py

@@ -0,0 +1,115 @@
+from http import HTTPStatus
+from typing import List
+
+import bs4
+
+from modules.zim import lib
+from wrolpi.common import logger, aiohttp_get, get_download_info, download_file, background_task
+from wrolpi.downloader import Downloader, Download, DownloadResult
+
+__all__ = ['KiwixCatalogDownloader', 'KiwixZimDownloader', 'kiwix_zim_downloader', 'kiwix_catalog_downloader']
+
+from wrolpi.files.lib import refresh_files
+from wrolpi.vars import PYTEST
+
+logger = logger.getChild(__name__)
+
+
+async def fetch_hrefs(url: str) -> List[str]:
+    content, status = await aiohttp_get(url, timeout=20)
+    if status != HTTPStatus.OK:
+        raise RuntimeError(f'Failed to fetch file catalog')
+
+    soup = bs4.BeautifulSoup(content, 'html.parser')
+    downloads = list()
+    for a_ in soup.find_all('a'):
+        downloads.append(a_['href'])
+    return downloads
+
+
+class KiwixCatalogDownloader(Downloader):
+    """
+    Downloads the catalog of Zim files from Kiwix.  Searches for the latest Zim that matches the provided URL.
+
+    If a newer file exists and has not been downloaded, schedule a Download.
+    """
+
+    name = 'kiwix_catalog'
+    listable = False
+
+    async def do_download(self, download: Download) -> DownloadResult:
+        url = download.url
+
+        # Get the parent directory of the file to be downloaded
+        *parents, name = url.split('/')
+        parent_url = '/'.join(parents)
+
+        # Find the latest Zim that has the correct name.
+        downloadable_files = await fetch_hrefs(parent_url)
+        matching_zims = sorted([i for i in downloadable_files if i.startswith(name) and i.endswith('.zim')])
+        if not matching_zims:
+            return DownloadResult(success=False, error=f'No Zim files match {str(repr(url))}')
+
+        logger.debug(f'Found matching Zims: {matching_zims}')
+        latest_zim = f'{parent_url}/{matching_zims[-1]}'
+
+        return DownloadResult(
+            success=True,
+            downloads=[latest_zim, ],
+        )
+
+
+kiwix_catalog_downloader = KiwixCatalogDownloader()
+
+
+class KiwixZimDownloader(Downloader):
+    """Downloads a Zim file to the zim directory."""
+    name = 'kiwix_zim'
+    listable = False
+
+    async def do_download(self, download: Download) -> DownloadResult:
+        url = download.url
+
+        download_info = await get_download_info(url)
+
+        zim_directory = lib.get_zim_directory()
+        zim_directory.mkdir(parents=True, exist_ok=True)
+        output_path = zim_directory / f'{download_info.name}'
+
+        if output_path.is_file():
+            output_size = output_path.stat().st_size
+            if download_info.size and download_info.size == output_size:
+                # File is already downloaded.
+                return DownloadResult(success=True)
+
+        logger.info(f'Downloading Zim {url} to {output_path} of size {download_info.size}')
+        await download_file(url, output_path=output_path, info=download_info)
+
+        # Notify the maintainer if outdated Zim files are lying around.
+        lib.flag_outdated_zim_files()
+        # Notify the maintainer to restart Kiwix container.
+        lib.flag_kiwix_restart()
+
+        output_size = output_path.stat().st_size
+        if download_info.size and output_size != download_info.size:
+            return DownloadResult(
+                success=False,
+                error=f'Download size does not match: {output_size} != {download_info.size}')
+
+        return_code = await lib.check_zim(output_path)
+        if return_code == 127:
+            logger.warning(f'Not validating {output_path} because zimcheck is not installed')
+        elif return_code > 0:
+            return DownloadResult(success=False, error=f'Zim file is invalid')
+
+        # Add the new Zim to the FileGroups, model it.
+        if PYTEST:
+            await refresh_files([output_path, ])
+        else:
+            background_task(refresh_files([output_path, ]))
+
+        # Location is just the generic Kiwix viewer.
+        return DownloadResult(success=True, location='/zim')
+
+
+kiwix_zim_downloader = KiwixZimDownloader()

modules/zim/kiwix.py

@@ -0,0 +1,134 @@
+WIKIPEDIA_LANGUAGES = ['ab', 'ace', 'ady', 'af', 'alt', 'am', 'ami', 'an', 'ang', 'anp', 'ar', 'arc', 'ary', 'arz',
+                       'as', 'ast', 'atj', 'av', 'avk', 'awa', 'ay', 'az', 'azb', 'ba', 'ban', 'bar', 'bcl', 'be',
+                       'be-tarask', 'bg', 'bho', 'bi', 'bjn', 'blk', 'bm', 'bn', 'bo', 'bpy', 'br', 'bs', 'bug', 'bxr',
+                       'ca', 'cbk', 'cdo', 'ce', 'ceb', 'ch', 'chr', 'chy', 'ckb', 'co', 'cr', 'crh', 'cs', 'csb', 'cu',
+                       'cv', 'cy', 'da', 'dag', 'de', 'din', 'diq', 'dsb', 'dty', 'dv', 'dz', 'ee', 'el', 'eml', 'en',
+                       'eo', 'es', 'et', 'eu', 'ext', 'fa', 'fat', 'ff', 'fi', 'fj', 'fo', 'fr', 'frp', 'frr', 'fur',
+                       'fy', 'ga', 'gag', 'gan', 'gcr', 'gd', 'gl', 'glk', 'gn', 'gom', 'gor', 'got', 'gsw', 'gu',
+                       'guc', 'gur', 'guw', 'gv', 'ha', 'hak', 'haw', 'he', 'hi', 'hif', 'hr', 'hsb', 'ht', 'hu', 'hy',
+                       'hyw', 'ia', 'id', 'ie', 'ig', 'ik', 'ilo', 'inh', 'io', 'is', 'it', 'iu', 'ja', 'jam', 'jbo',
+                       'jv', 'ka', 'kaa', 'kab', 'kbd', 'kbp', 'kcg', 'kg', 'ki', 'kk', 'kl', 'km', 'kn', 'ko', 'koi',
+                       'krc', 'ks', 'ksh', 'ku', 'kv', 'kw', 'ky', 'la', 'lad', 'lb', 'lbe', 'lez', 'lfn', 'lg', 'li',
+                       'lij', 'lld', 'lmo', 'ln', 'lo', 'lt', 'ltg', 'lv', 'lzh', 'mad', 'mai', 'map-bms', 'mdf', 'mg',
+                       'mhr', 'mi', 'min', 'mk', 'ml', 'mn', 'mni', 'mnw', 'mr', 'mrj', 'ms', 'mt', 'mwl', 'my', 'myv',
+                       'mzn', 'na', 'nah', 'nan', 'nap', 'nb', 'nds', 'nds-nl', 'ne', 'new', 'nia', 'nl', 'nn', 'nov',
+                       'nrm', 'nso', 'nv', 'ny', 'oc', 'olo', 'om', 'or', 'os', 'pa', 'pag', 'pam', 'pap', 'pcd', 'pdc',
+                       'pfl', 'pi', 'pih', 'pl', 'pms', 'pnb', 'pnt', 'ps', 'pt', 'pwn', 'qu', 'rm', 'rmy', 'rn', 'ro',
+                       'roa-tara', 'ru', 'rue', 'rup', 'rw', 'sa', 'sah', 'sat', 'sc', 'scn', 'sco', 'sd', 'se', 'sg',
+                       'sgs', 'sh', 'shi', 'shn', 'si', 'sk', 'skr', 'sl', 'sm', 'smn', 'sn', 'so', 'sq', 'sr', 'srn',
+                       'ss', 'st', 'stq', 'su', 'sv', 'sw', 'szl', 'szy', 'ta', 'tay', 'tcy', 'te', 'tet', 'tg', 'th',
+                       'ti', 'tk', 'tl', 'tn', 'to', 'tpi', 'tr', 'trv', 'ts', 'tt', 'tum', 'tw', 'ty', 'tyv', 'udm',
+                       'ug', 'uk', 'ur', 'uz', 've', 'vec', 'vep', 'vi', 'vls', 'vo', 'vro', 'wa', 'war', 'wo', 'wuu',
+                       'xal', 'xh', 'xmf', 'yi', 'yo', 'yue', 'za', 'zea', 'zh', 'zu']
+
+WIKITIONARY_LANGUAGES = ['af', 'am', 'an', 'ang', 'ar', 'ast', 'ay', 'az', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'chr',
+                         'co', 'cs', 'csb', 'cy', 'da', 'de', 'dv', 'el', 'en', 'eo', 'es', 'et', 'eu', 'fa', 'fi',
+                         'fj', 'fo', 'fr', 'fy', 'ga', 'gd', 'gl', 'gn', 'gu', 'guw', 'gv', 'he', 'hi', 'hif', 'hr',
+                         'hsb', 'hu', 'hy', 'ia', 'id', 'ie', 'io', 'is', 'it', 'iu', 'ja', 'jbo', 'jv', 'ka', 'kk',
+                         'kl', 'km', 'kn', 'ko', 'ks', 'ku', 'kw', 'ky', 'la', 'lb', 'li', 'ln', 'lo', 'lt', 'lv', 'mg',
+                         'mi', 'mk', 'ml', 'mn', 'mr', 'ms', 'mt', 'my', 'na', 'nah', 'nan', 'nb', 'nds', 'ne', 'nl',
+                         'nn', 'oc', 'or', 'pa', 'pl', 'pnb', 'ps', 'pt', 'qu', 'ro', 'ru', 'rup', 'rw', 'sa', 'scn',
+                         'sd', 'sg', 'sh', 'si', 'sk', 'sl', 'sm', 'so', 'sq', 'sr', 'ss', 'st', 'su', 'sv', 'sw', 'ta',
+                         'te', 'tg', 'th', 'ti', 'tk', 'tl', 'tn', 'tpi', 'tr', 'ts', 'tt', 'ug', 'uk', 'ur', 'uz',
+                         'vec', 'vi', 'vo', 'wa', 'wo', 'yi', 'zh', 'zu']
+
+WIKIBOOKS_LANGAUGES = ['af', 'ar', 'az', 'ba', 'be', 'bg', 'bn', 'bs', 'ca', 'cs', 'cv', 'cy', 'da', 'de', 'el', 'en',
+                       'eo', 'es', 'et', 'eu', 'fa', 'fi', 'fr', 'fy', 'gl', 'he', 'hi', 'hr', 'hu', 'hy', 'ia', 'id',
+                       'is', 'it', 'ja', 'ka', 'kk', 'km', 'ko', 'ku', 'ky', 'la', 'li', 'lt', 'mg', 'mk', 'ml', 'mr',
+                       'ms', 'nb', 'ne', 'nl', 'oc', 'pa', 'pl', 'pt', 'ro', 'ru', 'sa', 'si', 'sk', 'sl', 'sq', 'sr',
+                       'sv', 'ta', 'te', 'tg', 'th', 'tl', 'tr', 'tt', 'uk', 'ur', 'vi', 'zh']
+
+KIWIX_CATALOG = [
+    {
+        'name': 'Wikibooks (with images)',
+        'languages': WIKIBOOKS_LANGAUGES,
+        'size': 4.3 * 1073741824,
+    },
+    {
+        'name': 'Wikibooks (no images)',
+        'languages': WIKIBOOKS_LANGAUGES,
+        'size': 3.4 * 1073741824,
+    },
+    {
+        'name': 'Wikipedia (with images)',
+        'languages': WIKIPEDIA_LANGUAGES,
+        'size': 94 * 1073741824,
+    },
+    {
+        'name': 'Wikipedia (mini)',
+        'languages': WIKIPEDIA_LANGUAGES,
+        'size': 13 * 1073741824,
+    },
+    {
+        'name': 'Wikipedia (no images)',
+        'languages': WIKIPEDIA_LANGUAGES,
+        'size': 50 * 1073741824,
+    },
+    {
+        'name': 'Wikiversity (with images)',
+        'languages': ['ar', 'cs', 'de', 'el', 'en', 'es', 'fi', 'fr', 'hi', 'it', 'ja', 'ko', 'pt', 'ru', 'sl', 'sv',
+                      'zh'],
+        'size': 2.3 * 1073741824,
+    },
+    {
+        'name': 'Wikiversity (no images)',
+        'languages': ['ar', 'cs', 'de', 'el', 'en', 'es', 'fi', 'fr', 'hi', 'it', 'ja', 'ko', 'pt', 'ru', 'sl', 'sv',
+                      'zh'],
+        'size': 1.8 * 1073741824,
+    },
+    {
+        'name': 'Wikitionary (no images)',
+        'languages': WIKITIONARY_LANGUAGES,
+        'size': 7.3 * 1073741824,
+    },
+    {
+        'name': 'Wikitionary (with images)',
+        'languages': WIKITIONARY_LANGUAGES,
+        'size': 8.0 * 1073741824,
+    },
+    {
+        'name': 'Wikisource',
+        'languages': ['ang', 'ar', 'as', 'az', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'cs', 'cy', 'da', 'de', 'el', 'en',
+                      'eo', 'es', 'et', 'eu', 'fa', 'fi', 'fo', 'fr', 'gl', 'gu', 'he', 'hr', 'ht', 'hu', 'hy', 'id',
+                      'is', 'it', 'ja', 'kn', 'ko', 'la', 'li', 'lt', 'mk', 'ml', 'mr', 'nan', 'nb', 'nl', 'or', 'pa',
+                      'pl', 'pms', 'pt', 'ro', 'ru', 'sa', 'sah', 'sk', 'sl', 'sr', 'sv', 'ta', 'te', 'th', 'tr', 'uk',
+                      'vec', 'vi', 'yi', 'zh', 'zh-min-nan'],
+        'size': 16 * 1073741824,
+    },
+    {
+        'name': 'Raspberry Pi (Stack Exchange)',
+        'languages': ['en'],
+        'size': 300 * 1048576,
+    },
+    {
+        'name': 'Vikidia',
+        'languages': ['ca', 'en', 'es', 'eu', 'fr', 'it', 'ru', 'scn'],
+        'size': 711 * 1048576,
+    },
+    {
+        'name': 'Stackoverflow (Stack Exchange)',
+        'languages': ['en', 'es', 'ja', 'pt', 'ru'],
+        'size': 76 * 1073741824,
+    },
+    {
+        'name': 'Superuser (Stack Exchange)',
+        'languages': ['en'],
+        'size': 3 * 1073741824,
+    },
+    {
+        'name': 'iFixit',
+        'languages': ['de', 'en', 'es', 'fr', 'it', 'ja', 'ko', 'nl', 'pt', 'ru', 'tr', 'zh'],
+        'size': 3 * 1073741824,
+    },
+    {
+        'name': 'Gutenberg',
+        'languages': ['af', 'ale', 'ang', 'ar', 'arp', 'bg', 'bgs', 'bo', 'br', 'brx', 'ca', 'ceb', 'cs', 'csb', 'cy',
+                      'da', 'de', 'el', 'en', 'enm', 'eo', 'es', 'et', 'fa', 'fi', 'fr', 'fur', 'fy', 'ga', 'gl', 'gla',
+                      'grc', 'he', 'hu', 'ia', 'ilo', 'is', 'it', 'iu', 'ja', 'kha', 'kld', 'ko', 'la', 'mi', 'mul',
+                      'myn', 'nah', 'nai', 'nap', 'nav', 'nl', 'no', 'oc', 'oji', 'pl', 'pt', 'rmr', 'ro', 'ru', 'sa',
+                      'sl', 'sr', 'sv', 'te', 'tl', 'zh'],
+        'size': 70 * 1073741824,
+    },
+]
+
+KIWIX_CATALOG_BY_NAME = {i['name']: i for i in KIWIX_CATALOG}

modules/zim/lib.py

@@ -0,0 +1,484 @@
+import asyncio
+import functools
+import pathlib
+import re
+from datetime import datetime
+from typing import List, Tuple, Dict
+
+from libzim import Entry
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.exc import NoResultFound  # noqa
+
+from modules.zim import kiwix
+from modules.zim.kiwix import KIWIX_CATALOG
+from modules.zim.models import Zim, Zims, TagZimEntry, ZimSubscription
+from wrolpi import flags
+from wrolpi.common import register_modeler, logger, extract_html_text, extract_headlines, get_media_directory, walk, \
+    register_refresh_cleanup, background_task
+from wrolpi.db import get_db_session, optional_session, get_db_curs
+from wrolpi.downloader import DownloadFrequency
+from wrolpi.errors import UnknownZim, UnknownZimSubscription
+from wrolpi.files.lib import refresh_files
+from wrolpi.files.models import FileGroup
+from wrolpi.vars import PYTEST, DOCKERIZED
+
+logger = logger.getChild(__name__)
+
+__all__ = [
+    'flag_outdated_zim_files',
+    'get_all_entries_tags',
+    'get_entries_tags',
+    'get_entry',
+    'get_kiwix_catalog',
+    'get_unique_paths',
+    'get_zim',
+    'get_zims',
+    'zim_modeler',
+    'zim_download_url_to_name',
+]
+
+
+@register_modeler
+async def zim_modeler():
+    while True:
+        with get_db_session(commit=True) as session:
+            file_groups = session.query(FileGroup, Zim) \
+                .filter(FileGroup.indexed != True,  # noqa
+                        FileGroup.primary_path.ilike('%.zim'),
+                        ) \
+                .outerjoin(Zim, Zim.file_group_id == FileGroup.id) \
+                .limit(10)
+            file_groups: List[Tuple[FileGroup, Zim]] = list(file_groups)
+
+            processed = 0
+            for file_group, zim in file_groups:
+                processed += 1
+
+                zim_id = None
+                try:
+                    if not zim:
+                        zim = Zim(file_group=file_group, path=file_group.primary_path)
+                        session.add(zim)
+                        session.flush([zim])
+                    zim_id = zim.id
+                    file_group.indexed = True
+                except Exception as e:
+                    if PYTEST:
+                        raise
+                    logger.error(f'Unable to model Zim {zim_id=} {file_group.primary_path=}', exc_info=e)
+
+            logger.debug(f'Modeled {processed} zim files')
+
+            if processed < 10:
+                # Did not reach limit, do not query again.
+                break
+
+        # Sleep to catch cancel.
+        await asyncio.sleep(0)
+
+
+def get_all_entries_tags():
+    """Returns a dict of dicts which contain the names of all Tags of each Entry's path.
+
+    Example:
+        {
+            1: {
+                    'path1': ['tag1', 'tag2'],
+            },
+            2: {
+                    'path2': ['tag1'],
+            },
+        """
+    stmt = '''
+        SELECT tz.zim_id, tz.zim_entry, array_agg(t.name)::TEXT[]
+        FROM
+            tag t
+            LEFT JOIN tag_zim tz on t.id = tz.tag_id
+        GROUP BY 1, 2
+    '''
+    with get_db_curs() as curs:
+        curs.execute(stmt)
+        entries = dict()
+        for zim_id, zim_entry, tag_names in curs.fetchall():
+            if not zim_id:
+                continue
+            try:
+                entries[zim_id][zim_entry] = tag_names
+            except KeyError:
+                entries[zim_id] = {zim_entry: tag_names}
+    return entries
+
+
+def get_entries_tags(paths: List[str], zim_id: int):
+    all_entries_tags = get_all_entries_tags()
+    entries = dict()
+    for path in paths:
+        zim_tag_entries = all_entries_tags.get(zim_id, dict())
+        entries[path] = zim_tag_entries.get(path, list())
+
+    return entries
+
+
+def get_unique_paths(*paths: str) -> List[str]:
+    """Return a new list which contains unique Zim paths."""
+    new_paths = list()
+    simple_paths = list()
+    for path in paths:
+        simple_path = path.replace('-', '').replace('_', '').replace('/', '').lower()
+        if simple_path not in simple_paths:
+            new_paths.append(path)
+            simple_paths.append(simple_path)
+    return new_paths
+
+
+@optional_session
+def headline_zim(search_str: str, zim_id: int, tag_names: List[str] = None, offset: int = 0, limit: int = 10,
+                 session: Session = None) -> Dict:
+    zim_id = int(zim_id)
+    zim = get_zim(zim_id, session=session)
+
+    if tag_names:
+        # TODO what is the count goes over the limit?
+        entries = zim.entries_with_tags(tag_names, offset=offset, limit=1_000, session=session)
+        estimate = 0
+    elif search_str:
+        # Get suggested results (this searches titles).
+        results = zim.suggest(search_str, offset, limit)
+        search_results = zim.search(search_str, offset, limit)
+        # Remove duplicate entries (disambiguation or redirect).
+        results = get_unique_paths(*results, *search_results)
+        results = results[:limit]
+        entries = [zim.get_entry(i) for i in results]
+        estimate = zim.estimate(search_str)
+    else:
+        raise RuntimeError('Must provide search_str or tag_names.')
+
+    articles = [bytes(i.get_item().content).decode('UTF-8') for i in entries]
+    articles = [extract_html_text(i) for i in articles]
+
+    entries_tag_names = get_entries_tags([i.path for i in entries], zim_id)
+
+    # Always get headlines even if there is no search_str.  This is because Postgres will return the start of each
+    # article.
+    headlines = extract_headlines(articles, search_str)
+
+    search = list()
+    if search_str:
+        # `search_str` allows us to headline the title.
+        titles = [i.title for i in entries]
+        headline_titles = extract_headlines(titles, search_str)
+
+        for entry in entries:
+            title_headline, title_rank = headline_titles.pop(0)
+            headline, rank = headlines.pop(0)
+            if rank > 0 or title_rank > 0:
+                # The result for each Entry.
+                path = entry.path
+                entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list()
+                search.append(dict(
+                    headline=headline,
+                    path=path,
+                    rank=max(rank, title_rank),
+                    tag_names=entry_tag_names,
+                    title=title_headline,
+                ))
+                if tag_names:
+                    estimate += 1
+    else:
+        # No `search_str`, we return all entries.
+        for entry in entries:
+            headline, _ = headlines.pop(0)
+            path = entry.path
+            entry_tag_names = entries_tag_names[path] if path in entries_tag_names else list()
+            search.append(dict(
+                headline=headline,
+                path=path,
+                rank=0,
+                tag_names=entry_tag_names,
+                title=entry.title,
+            ))
+        estimate = len(entries)
+
+    results = dict(
+        path=zim.path,
+        metadata=zim.zim_metadata,
+        search=search,
+        estimate=estimate,
+    )
+    return results
+
+
+@optional_session
+def get_zim(zim_id: int, session: Session = None) -> Zim:
+    """Return the Zim record of the provided path.
+
+    @raise UnknownZim: When no record exists with the id.
+    @warning: Results are cached."""
+    try:
+        zim: Zim = session.query(Zim) \
+            .join(FileGroup, FileGroup.id == Zim.file_group_id) \
+            .filter(Zim.id == zim_id) \
+            .one()
+        return zim
+    except Exception as e:
+        raise UnknownZim() from e
+
+
+@functools.lru_cache(maxsize=1_000)
+def get_entry(path: str, zim_id: int) -> Entry:
+    with get_db_session() as session:
+        zim = get_zim(zim_id, session=session)
+        entry = zim.get_entry(path, throw=True)
+    return entry
+
+
+@optional_session
+def get_zims(ids: List[int] = None, session: Session = None) -> List[Zim]:
+    if ids:
+        zims = session.query(Zim).filter(Zim.id.in_(ids)).all()
+    else:
+        zims = Zims.get_all(session=session)
+
+    return zims
+
+
+@optional_session(commit=True)
+def delete_zims(ids: List[int], session: Session = None):
+    zims = get_zims(ids, session=session)
+    for zim in zims:
+        zim.delete()
+
+
+@optional_session
+async def add_tag(tag_name: str, zim_id: int, zim_entry: str, session: Session = None) -> TagZimEntry:
+    zim = session.query(Zim).filter_by(id=zim_id).one()
+    tag_zim_entry = zim.tag_entry(tag_name, zim_entry)
+    session.commit()
+    return tag_zim_entry
+
+
+@optional_session
+async def untag(tag_name: str, zim_id: int, zim_entry: str, session: Session = None):
+    zim: Zim = session.query(Zim).filter_by(id=zim_id).one()
+    zim.untag_entry(tag_name, zim_entry)
+    session.commit()
+
+
+def get_kiwix_catalog():
+    return KIWIX_CATALOG.copy()
+
+
+@optional_session
+def get_kiwix_subscriptions(session: Session = None) -> Dict[str, ZimSubscription]:
+    subscriptions = session.query(ZimSubscription)
+    subscriptions_by_name = {i.name: i for i in subscriptions}
+    return subscriptions_by_name
+
+
+@optional_session(commit=True)
+async def subscribe(name: str, language: str, session: Session = None,
+                    frequency: int = DownloadFrequency.days180) -> ZimSubscription:
+    subscription = ZimSubscription.get_or_create(name, session=session)
+    subscription.language = language
+
+    if name == 'Wikipedia (with images)':
+        url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_maxi_'
+    elif name == 'Wikipedia (mini)':
+        url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_mini_'
+    elif name == 'Wikipedia (no images)':
+        url = f'https://download.kiwix.org/zim/wikipedia/wikipedia_{language}_all_nopic_'
+    elif name == 'Wikiversity (with images)':
+        url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_maxi_'
+    elif name == 'Wikiversity (no images)':
+        url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_nopic_'
+    elif name == 'Wikitionary (no images)':
+        url = f'https://download.kiwix.org/zim/wiktionary/wiktionary_{language}_all_nopic_'
+    elif name == 'Wikitionary (with images)':
+        url = f'https://download.kiwix.org/zim/wikiversity/wikiversity_{language}_all_maxi_'
+    elif name == 'Wikibooks (no images)':
+        url = f'https://download.kiwix.org/zim/wikibooks/wikibooks_{language}_all_nopic_'
+    elif name == 'Wikibooks (with images)':
+        url = f'https://download.kiwix.org/zim/wikibooks/wikibooks_{language}_all_maxi_'
+    elif name == 'Wikisource':
+        url = f'https://download.kiwix.org/zim/wikisource/wikisource_{language}_all_maxi_'
+    elif name == 'Raspberry Pi (Stack Exchange)':
+        url = f'https://download.kiwix.org/zim/stack_exchange/raspberrypi.stackexchange.com_{language}_all_'
+    elif name == 'Vikidia':
+        url = f'https://download.kiwix.org/zim/vikidia/vikidia_{language}_all_maxi_'
+    elif name == 'Stackoverflow (Stack Exchange)':
+        url = f'https://download.kiwix.org/zim/stack_exchange/{language}.stackoverflow.com_{language}_all_'
+    elif name == 'Superuser (Stack Exchange)':
+        url = f'https://download.kiwix.org/zim/stack_exchange/superuser.com_{language}_all_'
+    elif name == 'iFixit':
+        url = f'https://download.kiwix.org/zim/ifixit/ifixit_{language}_all_'
+    elif name == 'Gutenberg':
+        url = f'https://download.kiwix.org/zim/gutenberg/gutenberg_{language}_all_'
+    else:
+        raise ValueError(f'{name} not a valid Kiwix subscription name!')
+
+    if language not in kiwix.KIWIX_CATALOG_BY_NAME[name]['languages']:
+        raise ValueError(f'Language {repr(str(language))} is invalid for {name}')
+
+    subscription.change_download(url, frequency, session=session)
+    session.add(subscription)
+    session.flush([subscription])
+    session.commit()
+    return subscription
+
+
+@optional_session
+async def unsubscribe(subscription_id: int, session: Session = None):
+    subscription: ZimSubscription = session.query(ZimSubscription).filter_by(id=subscription_id).one_or_none()
+    if subscription:
+        session.delete(subscription)
+        if subscription.download:
+            session.delete(subscription.download)
+        session.commit()
+    else:
+        raise UnknownZimSubscription(f'{subscription_id=}')
+
+
+KIWIX_URL_PARSER = re.compile(r'https:\/\/download\.kiwix\.org\/zim\/(.+?)\/(.+?)_(\w{2,3})(_.+)')
+
+
+def zim_download_url_to_name(url: str) -> Tuple[str, str]:
+    _, project, language, flavor = KIWIX_URL_PARSER.match(url).groups()
+    if project == 'wikipedia' and flavor == '_all_maxi_':
+        name = 'Wikipedia (with images)'
+    elif project == 'wikipedia' and flavor == '_all_nopic_':
+        name = 'Wikipedia (no images)'
+    elif project == 'wikipedia' and flavor == '_all_mini_':
+        name = 'Wikipedia (mini)'
+    elif project == 'wikiversity' and flavor == '_all_maxi_':
+        name = 'Wikiversity (with images)'
+    elif project == 'wikiversity' and flavor == '_all_nopic_':
+        name = 'Wikiversity (no images)'
+    elif project == 'wiktionary' and flavor == '_all_maxi_':
+        name = 'Wikitionary (with images)'
+    elif project == 'wiktionary' and flavor == '_all_nopic_':
+        name = 'Wikitionary (no images)'
+    elif project == 'wikibooks' and flavor == '_all_maxi_':
+        name = 'Wikibooks (with images)'
+    elif project == 'wikibooks' and flavor == '_all_nopic_':
+        name = 'Wikibooks (no images)'
+    elif project == 'wikisource' and flavor == '_all_maxi_':
+        name = 'Wikisource'
+    elif project == 'raspberrypi.stackexchange.com' and flavor == '_all_':
+        name = 'Raspberry Pi (Stack Exchange)'
+    elif project == 'vikidia' and flavor == '_all_maxi_':
+        name = 'Vikidia'
+    elif 'stackoverflow.com' in project and flavor == '_all_':
+        name = 'Stackoverflow (Stack Exchange)'
+    elif project == 'superuser.com' and flavor == '_all_':
+        name = 'Superuser (Stack Exchange)'
+    elif project == 'ifixit' and flavor == '_all_':
+        name = 'iFixit'
+    elif project == 'gutenberg' and flavor == '_all_':
+        name = 'Gutenberg'
+    else:
+        raise RuntimeError(f'Could not find name for Zim URL: {url} {project=} {flavor=}')
+
+    if language not in kiwix.KIWIX_CATALOG_BY_NAME[name]['languages']:
+        raise ValueError(f'Language {repr(str(language))} is invalid for {name}')
+
+    return name, language
+
+
+def get_zim_directory() -> pathlib.Path:
+    media_directory = get_media_directory()
+    zim_directory = media_directory / 'zims'
+    return zim_directory
+
+
+async def check_zim(path: pathlib.Path) -> int:
+    """
+    Runs `zimcheck` against the Zim file.  Zim is valid only if this returns `0`.
+
+    @warning: Only performs the checksum check!
+    """
+    cmd = f'zimcheck -C {path.absolute()}'
+    proc = await asyncio.create_subprocess_shell(cmd)
+    stdout, stderr = await proc.communicate()
+    logger.debug(f'zimcheck returned {proc.returncode}')
+    if proc.returncode != 0 and stderr:
+        logger.debug(stderr)
+    return proc.returncode
+
+
+ZIM_NAME_PARSER = re.compile(r'(.+?)_(\d{4}-\d{2}).zim')
+
+
+def parse_name(path: pathlib.Path) -> Tuple[str, datetime]:
+    name, date = ZIM_NAME_PARSER.match(path.name).groups()
+    date = datetime.strptime(date, '%Y-%m')
+    return name, date
+
+
+def find_outdated_zim_files(path: pathlib.Path = None) -> Tuple[List[pathlib.Path], List[pathlib.Path]]:
+    """Search the Zim directory for outdated Zim files.  Returns a list of outdated, and a list of current Zim files."""
+    path = path or get_zim_directory()
+    if not path.is_dir():
+        logger.error(f'Cannot search for outdated Zim files because the directory does not exist: {path}')
+        return list(), list()
+
+    # Find all non-empty Zim files.
+    files = [i for i in walk(path) if i.is_file() and i.stat().st_size and i.suffix == '.zim']
+    zims = list()
+    for file in files:
+        try:
+            name, date = parse_name(file)
+        except Exception as e:
+            logger.warning(f'Found Zim {file} but its name does not match.', exc_info=e)
+            continue
+
+        zims.append((name, date, file))
+
+    # Order by newest first.
+    zims = sorted(zims, key=lambda i: i[1], reverse=True)
+    outdated_files = list()
+    current_files = list()
+    newest_names = list()
+    for name, date, file in zims:
+        if name in newest_names:
+            # Already saw this name, this date is older.
+            outdated_files.append(file)
+        else:
+            newest_names.append(name)
+            current_files.append(file)
+
+    return sorted(outdated_files), sorted(current_files)
+
+
+async def remove_outdated_zim_files(path: pathlib.Path = None):
+    """Deletes all old Zim files."""
+    outdated, _ = find_outdated_zim_files(path)
+    logger.info(f'Deleting {len(outdated)} outdated Zim files: {outdated}')
+    for file in outdated:
+        file.unlink()
+
+    if PYTEST:
+        await refresh_files(outdated)
+    else:
+        background_task(refresh_files(outdated))
+
+    flag_kiwix_restart()
+
+
+@register_refresh_cleanup
+def flag_outdated_zim_files():
+    """Set the `outdated_zims` Flag if outdated Zim files can be found, otherwise clear it."""
+    outdated, _ = find_outdated_zim_files()
+    if outdated:
+        logger.info('Outdated Zims were found')
+        flags.outdated_zims.set()
+    else:
+        logger.info('Outdated Zims were not found')
+        flags.outdated_zims.clear()
+
+
+def flag_kiwix_restart():
+    """Set `kiwix_restart` Flag only if running in Docker."""
+    if DOCKERIZED:
+        flags.kiwix_restart.set()
+    else:
+        logger.debug(f'Not setting kiwix_restart flag because this is not Dockerized')

modules/zim/models.py

@@ -0,0 +1,370 @@
+import dataclasses
+import functools
+import pathlib
+from collections import OrderedDict
+from datetime import datetime
+from typing import List, Tuple, OrderedDict as OrderedDictType, Dict, Optional, Set
+
+from libzim import Archive, Searcher, Query, Entry, SuggestionSearcher
+from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_
+from sqlalchemy.orm import relationship, Session
+from sqlalchemy.orm.exc import NoResultFound  # noqa
+
+from wrolpi import dates, tags
+from wrolpi.common import Base, extract_html_text, logger, get_relative_to_media_directory
+from wrolpi.dates import TZDateTime
+from wrolpi.db import optional_session, get_db_curs
+from wrolpi.downloader import Download, download_manager
+from wrolpi.errors import UnknownZimEntry, UnknownZimTagEntry
+from wrolpi.files.models import FileGroup
+from wrolpi.media_path import MediaPathType
+from wrolpi.tags import Tag, tag_names_to_zim_sub_select
+
+logger = logger.getChild(__name__)
+
+
+@dataclasses.dataclass
+class ZimMetadata:
+    date: str
+    creator: str
+    description: str
+    name: str
+    publisher: str
+    tags: str
+    title: str
+
+    def __json__(self) -> Dict:
+        d = dict(
+            date=self.date,
+            creator=self.creator,
+            description=self.description,
+            name=self.name,
+            publisher=self.publisher,
+            tags=self.tags,
+            title=self.title,
+        )
+        return d
+
+
+class EntrySummary:
+    zim_id: int
+    path: str
+    title: str
+
+    def __json__(self) -> Dict:
+        d = dict(
+            zim_id=self.zim_id,
+            path=self.path,
+            title=self.title,
+        )
+        return d
+
+
+class Zim(Base):
+    """Records of Zim files.
+
+    Zim files stores wiki content (and much more) for offline usage. Zim files are compressed data that can be searched.
+    """
+
+    __tablename__ = 'zim'
+    id = Column(Integer, primary_key=True)
+    path: pathlib.Path = Column(MediaPathType, nullable=False)
+
+    file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
+    file_group: FileGroup = relationship('FileGroup')
+
+    def __repr__(self):
+        return f'<Zim id={self.id} file_group_id={self.file_group_id} path={self.path}>'
+
+    def __json__(self) -> dict:
+        d = dict(
+            id=self.id,
+            path=self.path,
+            file_group_id=self.file_group_id,
+            metadata=self.zim_metadata,
+            size=self.file_group.size,
+        )
+        return d
+
+    @staticmethod
+    def from_paths(session: Session, *paths: pathlib.Path) -> 'Zim':
+        file_group = FileGroup.from_paths(session, *paths)
+        zim = Zim(path=file_group.primary_path, file_group=file_group)
+        session.add(zim)
+        session.flush([file_group, zim])
+        return zim
+
+    def get_zim(self) -> Archive:
+        if not self.path or not self.path.is_file():
+            raise FileNotFoundError(f'{self} does not exist!')
+
+        if not self.path.suffix != 'zim':
+            raise FileNotFoundError(f'{self} is not a valid zim file!')
+
+        zim = Archive(self.path)
+        return zim
+
+    def delete(self):
+        session: Session = Session.object_session(self)
+        self.path.unlink(missing_ok=True)
+        session.delete(self)
+
+    @functools.cached_property
+    def zim_metadata(self) -> ZimMetadata:
+        zim = self.get_zim()
+
+        def _get_metadata(path: str):
+            try:
+                return zim.get_metadata(path).decode()
+            except RuntimeError:
+                return None
+
+        metadata = ZimMetadata(
+            date=_get_metadata('Date'),
+            creator=_get_metadata('Creator'),
+            description=_get_metadata('Description'),
+            name=_get_metadata('Name'),
+            publisher=_get_metadata('Publisher'),
+            tags=_get_metadata('Tags'),
+            title=_get_metadata('Title'),
+        )
+        return metadata
+
+    def get_searcher(self) -> Searcher:
+        searcher = Searcher(self.get_zim())
+        return searcher
+
+    def get_search(self, search_str: str):
+        query = Query().set_query(search_str)
+        searcher = self.get_searcher()
+        search = searcher.search(query)
+        return search
+
+    def get_suggestion_searcher(self) -> SuggestionSearcher:
+        searcher = SuggestionSearcher(self.get_zim())
+        return searcher
+
+    def estimate(self, search_str: str) -> int:
+        """Count the number of Entry(s) that contain the `search_str`."""
+        search = self.get_search(search_str)
+        count = search.getEstimatedMatches()
+        return count
+
+    def search(self, search_str: str, offset: int = 0, limit: int = 10) -> List[str]:
+        """Return the paths of the Entry(s) that contain the `search_str`."""
+        search = self.get_search(search_str)
+        results = list(search.getResults(offset, limit))
+        return results
+
+    def suggest(self, search_str: str, offset: int = 0, limit: int = 10) -> List[str]:
+        """Return the paths of the Entry(s) that contain the `search_str`.  Also searches the titles."""
+        searcher = self.get_suggestion_searcher()
+        suggestions = searcher.suggest(search_str)
+        results = list(suggestions.getResults(offset, limit))
+        return results
+
+    def get_entry(self, path: str, throw: bool = True) -> Optional[Entry]:
+        """Search this Zim file for the provided Entry at the `path`."""
+        zim = self.get_zim()
+        try:
+            entry = zim.get_entry_by_path(path)
+        except KeyError:
+            if throw:
+                raise UnknownZimEntry(f'Cannot find entry at {path=}')
+            else:
+                return None
+        return entry
+
+    def get_entry_html(self, path: str) -> Optional[str]:
+        """Search this Zim file for the provided Entry at `path`, read its contents and return the HTML."""
+        content = bytes(self.get_entry(path).get_item().content).decode('UTF-8')
+        return content
+
+    def get_entry_text(self, path: str) -> Optional[str]:
+        """Search this Zim file for the provided Entry at `path`, read its contents and return the text in the HTML."""
+        html = self.get_entry_html(path)
+        content = extract_html_text(html)
+        return content
+
+    def tag_entry(self, tag_name: str, zim_entry: str) -> 'TagZimEntry':
+        """Create a TagZimEntry for this Zim at the provided entry (path) for the provided Tag."""
+        session = Session.object_session(self)
+        tag = Tag.find_by_name(tag_name, session=session)
+        tag_zim_entry = TagZimEntry(tag=tag, zim=self, zim_entry=zim_entry)
+        session.add(tag_zim_entry)
+        tags.schedule_save()
+        return tag_zim_entry
+
+    def untag_entry(self, tag_name: str, zim_entry: str):
+        """Removes any TagZimEntry of the Zim entry at the provided path for the provided Tag.
+
+        @raise UnknownZimTagEntry: No entry exists to remove."""
+        session = Session.object_session(self)
+        tag = Tag.find_by_name(tag_name, session=session)
+        tag_zim_entry = session.query(TagZimEntry).filter_by(
+            tag_id=tag.id,
+            zim_id=self.id,
+            zim_entry=zim_entry,
+        ).one_or_none()
+        if tag_zim_entry:
+            session.delete(tag_zim_entry)
+        else:
+            raise UnknownZimTagEntry(f'Could not find at {repr(str(zim_entry))}')
+        tags.schedule_save()
+
+    @optional_session
+    def entries_with_tags(self, tag_names: List[str], offset: int = 0, limit: int = 10, session: Session = None) \
+            -> List[Entry]:
+        """Return Zim Entries tagged with the provided names."""
+        tags_sub_select, params = tag_names_to_zim_sub_select(tag_names, zim_id=self.id)
+
+        with get_db_curs() as curs:
+            curs.execute(tags_sub_select, params)
+            zim_ids_entries: List[Tuple[int, int]] = list(curs.fetchall())
+
+        tag_zim_entries = session.query(TagZimEntry, Zim) \
+            .join(Zim, Zim.id == TagZimEntry.zim_id) \
+            .filter(tuple_(TagZimEntry.zim_id, TagZimEntry.zim_entry).in_(zim_ids_entries)) \
+            .order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry) \
+            .offset(offset).limit(limit) \
+            .distinct(TagZimEntry.zim_id, TagZimEntry.zim_entry)
+
+        entries = list()
+        for tag_zim_entry, zim in tag_zim_entries:
+            entry = zim.get_entry(tag_zim_entry.zim_entry)
+            entries.append(entry)
+
+        return entries
+
+    @functools.cached_property
+    def all_entries(self) -> Set[EntrySummary]:
+        """
+        Iterates over all Entries in the Archive.  Ignores non-article entries (javascript, images, etc).
+        """
+        zim = self.get_zim()
+        zim_id = self.id
+        entry_count = zim.all_entry_count
+        entries = set()
+        for entry_id in range(entry_count):
+            entry: Entry = zim._get_entry_by_id(entry_id)
+            title = entry.title
+            if title.startswith('m/') or title.startswith('-/') or title.startswith('I/'):
+                continue
+
+            summary = EntrySummary()
+            summary.zim_id = zim_id
+            summary.path = entry.path
+            summary.title = title
+            entries.add(summary)
+        return entries
+
+    def parse_name(self):
+        from modules.zim import lib
+        name, date = lib.parse_name(self.path)
+        return name, date
+
+
+class Zims:
+    """Convenience class which searches all known Zim files."""
+
+    @staticmethod
+    @optional_session
+    def get_all(session: Session = None) -> List[Zim]:
+        """Returns all Zim records."""
+        zims = list()
+        for zim in session.query(Zim).order_by(Zim.path):
+            try:
+                # It may not be possible to read the Zim file (drive not mounted), report and ignore.
+                zim.get_zim()
+                zims.append(zim)
+            except Exception as e:
+                logger.error(f'Failed to read Zim: {zim}', exc_info=e)
+        return zims
+
+    @classmethod
+    @optional_session
+    def estimate(cls, search_str: str, session: Session = None) -> OrderedDictType[Zim, int]:
+        zims = cls.get_all(session=session)
+        results = OrderedDict()
+        for zim in zims:
+            results[zim] = zim.estimate(search_str)
+        return results
+
+    @classmethod
+    @optional_session
+    def entries_with_tags(cls, tag_names: List[str], session: Session = None) -> OrderedDictType[Zim, int]:
+        zims = cls.get_all(session=session)
+        results = OrderedDict()
+        for zim in zims:
+            results[zim] = len(zim.entries_with_tags(tag_names, session=session))
+        return results
+
+
+class TagZimEntry(Base):
+    __tablename__ = 'tag_zim'
+
+    tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True)
+    tag: Tag = relationship('Tag')
+    zim_id = Column(Integer, ForeignKey('zim.id', ondelete='CASCADE'), primary_key=True)
+    zim: Zim = relationship('Zim')
+    zim_entry: str = Column(Text, nullable=False, primary_key=True)
+    created_at: datetime = Column(TZDateTime, default=dates.now)
+
+    def __repr__(self):
+        tag = self.tag_id
+        if self.tag:
+            tag = self.tag.name
+        zim = self.zim_id
+        if self.zim:
+            zim = get_relative_to_media_directory(self.zim.path)
+        return f'<TagZimEntry {tag=} {zim=} zim_entry={self.zim_entry}>'
+
+
+class ZimSubscription(Base):
+    __tablename__ = 'zim_subscription'
+
+    id: int = Column(Integer, primary_key=True)
+    name: str = Column(Text, unique=True, nullable=False)
+    language: str = Column(Text, nullable=False)
+    download_id: int = Column(Integer, ForeignKey('download.id', ondelete='CASCADE'), nullable=False)
+    download: Download = relationship('Download', primaryjoin='ZimSubscription.download_id==Download.id')
+
+    def __repr__(self):
+        name = self.name
+        language = self.language
+        download_id = self.download_id
+        return f'<ZimSubscription id={self.id} {name=} {language=} {download_id=}>'
+
+    def __json__(self):
+        d = dict(
+            id=self.id,
+            name=self.name,
+            language=self.language,
+            download_id=self.download_id,
+            download_url=self.download.url if self.download else None,
+        )
+        return d
+
+    def change_download(self, url: str, frequency: int, session: Session):
+        # A Zim file is found using the KiwixCatalogDownloader, then handled by the KiwixZimDownloader.
+        old_url = self.download.url if self.download else url
+        download = download_manager.get_or_create_download(old_url, session=session, reset_attempts=True)
+        download.url = url
+        download.frequency = frequency
+        download.downloader = 'kiwix_catalog'
+        download.sub_downloader = 'kiwix_zim'
+        download.attempts = 0
+        download.info_json = {'language': self.language, 'name': self.name}
+        session.flush([download, ])
+        self.download_id = download.id
+        return download
+
+    @staticmethod
+    @optional_session
+    def get_or_create(name: str, session: Session = None) -> 'ZimSubscription':
+        subscription = session.query(ZimSubscription).filter_by(name=name).one_or_none()
+        if subscription:
+            return subscription
+
+        subscription = ZimSubscription(name=name)
+        return subscription

modules/zim/schema.py

@@ -0,0 +1,95 @@
+from dataclasses import dataclass, field
+from typing import Optional, List, Dict
+
+
+@dataclass
+class ZimSearchRequest:
+    search_str: Optional[str] = None
+    tag_names: List[str] = field(default_factory=lambda: list())
+    offset: Optional[int] = 0
+    limit: Optional[int] = 10
+
+
+@dataclass
+class ZimMetadata:
+    path: str
+    creator: str
+    description: str
+    name: str
+    publisher: str
+    tags: str
+    title: str
+
+
+@dataclass
+class Zim:
+    id: int
+    path: str
+    file_group_id: int
+    metadata: ZimMetadata
+    size: int
+
+
+@dataclass
+class GetZimsResponse:
+    zims: List[Zim]
+
+
+@dataclass
+class ZimSearchHeadline:
+    headline: str
+    rank: float
+
+
+@dataclass
+class ZimSearchResult:
+    path: str
+    metadata: ZimMetadata
+    search: List[ZimSearchHeadline]
+
+
+@dataclass
+class ZimSearchResponse:
+    zim: ZimSearchResult
+
+
+@dataclass
+class TagZimEntry:
+    tag_name: str
+    zim_id: int
+    zim_entry: str
+
+
+@dataclass
+class ZimSubscription:
+    id: int
+    name: str
+    language: str
+    download_id: int
+    download_url: str
+
+
+@dataclass
+class ZimProject:
+    name: str
+    languages: List[str]
+    size: int
+
+
+@dataclass
+class ZimSubscriptions:
+    subscriptions: Dict[str, ZimSubscription]
+    catalog: List[ZimProject]
+    iso_639_codes: Dict[str, str]
+
+
+@dataclass
+class TagZimSubscribe:
+    name: str
+    language: str
+
+
+@dataclass
+class OutdatedZims:
+    outdated: List[str]
+    current: List[str]

modules/zim/test/test_api.py

@@ -0,0 +1,245 @@
+import json
+from http import HTTPStatus
+
+import pytest
+
+from modules.zim import lib
+from modules.zim.models import Zim
+from wrolpi.downloader import Download
+
+
+def test_zim_search(test_client, test_session, test_zim, tag_factory, assert_zim_search):
+    """Entries can be tagged and retrieved using their containing Zim."""
+    tag1, tag2 = tag_factory(), tag_factory()
+    test_zim.tag_entry(tag1.name, 'one')
+    test_zim.tag_entry(tag2.name, 'one')
+    test_zim.tag_entry(tag1.name, 'two')
+    test_session.commit()
+
+    # Test searching with only `search_str`.
+    assert_zim_search('one', 1, {
+        'path': test_zim.path,
+        'search': [
+            # Entry "One" contains a 'one' in the title, and the text.
+            {'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271},
+            # Home only contains one 'one'.
+            {'path': 'home', 'rank': 0.06079271, 'headline': 'file for testing.\n<b>One</b>', },
+        ],
+        'estimate': 2,
+    })
+    assert_zim_search('two', 1, {
+        'path': test_zim.path,
+        'search': [{'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271},
+                   {'path': 'home', 'headline': 'testing.\nOne\n<b>Two</b>', 'rank': 0.06079271}],
+        'estimate': 2,
+    })
+    assert_zim_search('item', 1, {
+        'path': test_zim.path,
+        'search': [{'path': 'one', 'headline': 'first <b>item</b>', 'rank': 0.06079271},
+                   {'path': 'two', 'headline': 'second <b>item</b>', 'rank': 0.06079271}],
+        'estimate': 2,
+    })
+
+    # Test searching with `search_str` and `tag_names`.
+    assert_zim_search('one', 1, tag_names=[tag2.name, ], expected={
+        'search': [
+            # Only entry "One" is tagged with Tag 2.
+            {'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271},
+        ],
+        'estimate': 1,
+    })
+    assert_zim_search('one', 1, tag_names=[tag1.name, tag2.name], expected={
+        'search': [
+            # Only "One" is tagged with Tag 1 and contains "one".
+            {'path': 'one', 'headline': 'This is the first item', 'rank': 0.06079271},
+        ],
+        'estimate': 1,
+    })
+    assert_zim_search('two', 1, tag_names=[tag1.name, ], expected={
+        'search': [
+            # Only "Two" was tagged with Tag 1 and contains "two".
+            {'path': 'two', 'headline': 'This is the second item', 'rank': 0.06079271},
+        ],
+        'estimate': 1,
+    })
+    assert_zim_search('two', 1, tag_names=[tag2.name, ], expected={
+        # Two was not tagged with Tag 1.
+        'search': [],
+        'estimate': 0,
+    })
+
+    # Test search with only `tag_names`.
+    assert_zim_search('', 1, tag_names=[tag1.name, tag2.name], expected={
+        'search': [
+            # Return all entries that are tagged with the Tags.
+            {'path': 'one', 'headline': 'This is the first item', 'rank': 0.0, 'tag_names': ['one', 'two']},
+        ],
+        'estimate': 1,
+    })
+
+
+def test_entries_by_tag(test_client, test_session, test_zim, tag_factory):
+    """Entries can be tagged and retrieved using their containing Zim."""
+    tag1, tag2 = tag_factory(), tag_factory()
+    test_zim.tag_entry(tag1.name, 'one')
+    test_zim.tag_entry(tag2.name, 'one')
+    test_zim.tag_entry(tag1.name, 'two')
+    test_session.commit()
+
+    result = test_zim.entries_with_tags([tag1.name, tag2.name])
+    assert [(i.path, i.title) for i in result] == [('one', 'One'), ]
+
+    result = test_zim.entries_with_tags([tag1.name, ])
+    assert [(i.path, i.title) for i in result] == [('one', 'One'), ('two', 'Two')]
+
+    result = test_zim.entries_with_tags([tag2.name, ])
+    assert [(i.path, i.title) for i in result] == [('one', 'One'), ]
+
+
+def test_zim_crud(test_client, test_session, test_directory, test_zim):
+    request, response = test_client.get('/api/zim')
+    assert response.status_code == HTTPStatus.OK
+    assert len(response.json['zims']) == 1
+    assert response.json['zims'][0]['path'] == test_zim.path.name
+
+    request, response = test_client.delete('/api/zim/1')
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    assert not test_zim.path.exists()
+    assert test_session.query(Zim).count() == 0
+
+
+def test_zim_tag_estimate(test_client, test_session, test_zim, tag_factory):
+    """Global search estimate returns Zim estimates."""
+    tag1, tag2 = tag_factory(), tag_factory()
+    test_session.commit()
+
+    content = dict(search_str=None, tag_names=[tag1.name, ])
+    request, response = test_client.post('/api/search_estimate', content=json.dumps(content))
+    assert response.status_code == HTTPStatus.OK
+    assert response.json['zims'] and response.json['zims'][0]['estimate'] == 0
+
+    # Entries can be tagged in the API.
+    content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
+    request, response = test_client.post('/api/zim/tag', content=json.dumps(content))
+    assert response.status_code == HTTPStatus.CREATED
+    # Create other Tags directly.
+    test_zim.tag_entry(tag2.name, 'one')
+    test_zim.tag_entry(tag1.name, 'two')
+    test_session.commit()
+
+    content = dict(search_str=None, tag_names=[tag1.name, ])
+    request, response = test_client.post('/api/search_estimate', content=json.dumps(content))
+    assert response.status_code == HTTPStatus.OK
+    assert response.json['zims'] and response.json['zims'][0]['estimate'] == 2
+
+
+def test_zim_tag_and_untag(test_client, test_session, test_zim, tag_factory):
+    """Zim entries can be Tagged and Untagged."""
+    tag1, tag2 = tag_factory(), tag_factory()
+    test_session.commit()
+
+    # Entries can be tagged in the API.
+    content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
+    request, response = test_client.post('/api/zim/tag', content=json.dumps(content))
+    assert response.status_code == HTTPStatus.CREATED
+
+    # Entries can be untagged in the API.
+    content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
+    request, response = test_client.post('/api/zim/untag', content=json.dumps(content))
+    assert response.status_code == HTTPStatus.NO_CONTENT
+
+    # Cannot untag twice.
+    content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
+    request, response = test_client.post('/api/zim/untag', content=json.dumps(content))
+    assert response.status_code == HTTPStatus.NOT_FOUND
+
+
+def test_zim_subscribe(test_session, test_client):
+    """A Kiwix subscription can be scheduled in the API.  The language can be changed."""
+    # Subscribe to English.
+    content = dict(name='Wikipedia (mini)', language='en')
+    request, response = test_client.post('/api/zim/subscribe', content=json.dumps(content))
+    assert response.status_code == HTTPStatus.CREATED
+
+    download: Download = test_session.query(Download).one()
+    assert download.url == 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_'
+    assert download.info_json == {'language': 'en', 'name': 'Wikipedia (mini)'}
+    assert download.frequency
+    request, response = test_client.get('/api/zim/subscribe')
+    assert response.status_code == HTTPStatus.OK
+    assert response.json['subscriptions'] == {
+        'Wikipedia (mini)': {'download_id': 1,
+                             'download_url': 'https://download.kiwix.org/zim/wikipedia/wikipedia_en_all_mini_',
+                             'id': 1,
+                             'language': 'en',
+                             'name': 'Wikipedia (mini)'}}
+
+    # Change subscription to French.
+    content = dict(name='Wikipedia (mini)', language='fr')
+    request, response = test_client.post('/api/zim/subscribe', content=json.dumps(content))
+    assert response.status_code == HTTPStatus.CREATED
+    assert test_session.query(Download).count() == 1
+
+    download: Download = test_session.query(Download).one()
+    assert download.url == 'https://download.kiwix.org/zim/wikipedia/wikipedia_fr_all_mini_'
+    assert download.info_json == {'language': 'fr', 'name': 'Wikipedia (mini)'}
+    assert download.frequency
+    request, response = test_client.get('/api/zim/subscribe')
+    assert response.status_code == HTTPStatus.OK
+    assert response.json['subscriptions'] == {
+        'Wikipedia (mini)': {'download_id': 1,
+                             'download_url': 'https://download.kiwix.org/zim/wikipedia/wikipedia_fr_all_mini_',
+                             'id': 1,
+                             'language': 'fr',
+                             'name': 'Wikipedia (mini)'}}
+
+    # Delete subscription.
+    request, response = test_client.delete('/api/zim/subscribe/1')
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    request, response = test_client.get('/api/zim/subscribe')
+    assert response.status_code == HTTPStatus.OK
+    assert response.json['subscriptions'] == {}
+    # No subscriptions or downloads.
+    assert test_session.query(lib.ZimSubscription).count() == 0
+    assert test_session.query(Download).count() == 0
+
+
+def test_get_zim_entry(test_session, test_client, test_zim):
+    request, response = test_client.get('/api/zim/1/entry/home')
+    assert response.status_code == HTTPStatus.OK
+
+    request, response = test_client.get('/api/zim/1/entry/does not exist')
+    assert response.status_code == HTTPStatus.NOT_FOUND
+
+
+@pytest.mark.asyncio
+async def test_find_outdated_zim_files(test_session, test_directory, test_zim_bytes, test_async_client):
+    """Outdated Zim files are found and reported.  They can automatically be deleted."""
+    # Can search empty directory.
+    outdated, current = lib.find_outdated_zim_files()
+    assert not outdated and not current
+
+    (test_directory / 'zims/books').mkdir(parents=True)
+
+    wikibooks_one = test_directory / 'zims/books/wikibooks_en_nopic_maxi_2023-01.zim'  # unique
+    wikipedia_one = test_directory / 'zims/wikipedia_en_all_maxi_2023-01.zim'  # outdated
+    wikipedia_two = test_directory / 'zims/wikipedia_en_all_maxi_2023-02.zim'  # keep this one
+    asdf = test_directory / 'zims/asdf.zim'  # Name cannot be parsed.
+    wikibooks_one.write_bytes(test_zim_bytes)
+    wikipedia_one.write_bytes(test_zim_bytes)
+    wikipedia_two.write_bytes(test_zim_bytes)
+    asdf.write_bytes(test_zim_bytes)
+    empty = test_directory / 'zims/wikipedia_en_all_maxi_2023-03.zim'  # empty files should be ignored.
+    empty.touch()
+    request, response = await test_async_client.get('/api/zim/outdated')
+    assert response.status_code == HTTPStatus.OK
+    assert response.json['outdated'] == [str(wikipedia_one.relative_to(test_directory)), ]
+
+    request, response = await test_async_client.delete('/api/zim/outdated')
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    # Only the outdated was deleted.
+    assert wikibooks_one.is_file()
+    assert not wikipedia_one.exists()
+    assert wikipedia_two.is_file()
+    assert asdf.is_file()
+    assert empty.is_file()

modules/zim/test/test_lib.py

@@ -0,0 +1,300 @@
+import datetime
+from http import HTTPStatus
+
+import pytest
+from libzim import Entry
+
+from modules.zim import lib
+from modules.zim.models import TagZimEntry, ZimSubscription
+from wrolpi import tags, flags
+from wrolpi.common import DownloadFileInfo
+from wrolpi.downloader import Download, import_downloads_config, save_downloads_config
+from wrolpi.errors import UnknownZim
+from wrolpi.files import lib as files_lib
+from wrolpi.files.models import FileGroup
+
+
+@pytest.mark.asyncio
+async def test_get_zim(test_session, zim_path_factory):
+    zim_path_factory()
+    await files_lib.refresh_files()
+
+    assert lib.get_zim(1)
+
+    with pytest.raises(UnknownZim):
+        assert lib.get_zim(2)
+
+
+@pytest.mark.asyncio
+async def test_zim_get_entry(test_session, zim_path_factory):
+    zim_path_factory()
+    await files_lib.refresh_files()
+
+    entry: Entry = lib.get_entry('one', 1)
+    assert entry.path == 'one'
+    assert entry.title == 'One'
+    assert bytes(entry.get_item().content).decode('UTF-8') == '''
+             <html>
+                 <head>
+                     <title>One</title>
+                 </head>
+                 <body>
+                     <h1>This is the first item</h1>
+                 </body>
+             </html>
+             '''
+
+    entry: Entry = lib.get_entry('home', 1)
+    assert entry.path == 'home'
+    assert entry.title == 'Homepage'
+
+
+@pytest.mark.asyncio
+async def test_zim_get_entries_tags(test_session, test_zim, tag_factory):
+    tag1, tag2 = tag_factory('tag1'), tag_factory('tag2')
+    test_zim.tag_entry(tag1.name, 'one')
+    test_zim.tag_entry(tag2.name, 'one')
+    test_zim.tag_entry(tag1.name, 'two')
+    test_session.commit()
+
+    assert lib.get_entries_tags(['one', 'two', 'home'], 1) \
+           == dict(one=['tag1', 'tag2'], two=['tag1'], home=[])
+
+
+def test_get_unique_paths():
+    paths = ['A/Prickly_Pear', 'A/Prickly_pear', 'A/Prickly-pear', 'A/Arborescent_Prickly_Pear',
+             'A/Arborescent_prickly_pear', 'A/Beavertail_prickly_pear', 'A/Brittle_prickly-pear',
+             'A/Chenille_prickly-pear', 'A/Coastal_Prickly_Pear', 'A/Coastal_prickly_pear',
+             'A/Prickly_pears_in_South_Africa', 'A/Prickly_Pear_Cays', 'A/Prickly_pears_in_Australia', 'A/Opuntia',
+             'A/Prickly_Pear_(British_Virgin_Islands)', 'A/Prickly_Pear_Island', 'A/Prickly_Pears_(film)',
+             'A/Invasive_succulent_plants_in_South_Africa', 'A/Cactoblastis_Memorial_Hall', 'A/Thomas_Harvey_Johnston']
+    assert lib.get_unique_paths(*paths) == ['A/Prickly_Pear',
+                                            'A/Arborescent_Prickly_Pear',
+                                            'A/Beavertail_prickly_pear',
+                                            'A/Brittle_prickly-pear',
+                                            'A/Chenille_prickly-pear',
+                                            'A/Coastal_Prickly_Pear',
+                                            'A/Prickly_pears_in_South_Africa',
+                                            'A/Prickly_Pear_Cays',
+                                            'A/Prickly_pears_in_Australia',
+                                            'A/Opuntia',
+                                            'A/Prickly_Pear_(British_Virgin_Islands)',
+                                            'A/Prickly_Pear_Island',
+                                            'A/Prickly_Pears_(film)',
+                                            'A/Invasive_succulent_plants_in_South_Africa',
+                                            'A/Cactoblastis_Memorial_Hall',
+                                            'A/Thomas_Harvey_Johnston']
+
+
+def test_zim_tags_config(test_session, test_directory, test_zim, tag_factory, test_tags_config, fake_now):
+    fake_now(datetime.datetime(2000, 1, 1, 0, 0, 0, 1))
+    config = tags.get_tags_config()
+    assert not config.tag_zims
+
+    # Tag three Zim entries.
+    tag1, tag2 = tag_factory('Tag1'), tag_factory('Tag2')
+    test_zim.tag_entry(tag1.name, 'one')
+    test_zim.tag_entry(tag2.name, 'one')
+    test_zim.tag_entry(tag1.name, 'two')
+    test_session.commit()
+    # Tags are created in empty database.
+    assert {i[0] for i in test_session.query(tags.Tag.id)} == {1, 2}
+
+    config = tags.get_tags_config()
+    assert config.tag_zims == [
+        ['Tag1', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
+        ['Tag2', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
+        ['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
+    ]
+
+    # Remove a tag, the config should change.
+    test_zim.untag_entry(tag1.name, 'one')
+    test_session.commit()
+
+    config = tags.get_tags_config()
+    assert config.tag_zims == [
+        ['Tag2', test_zim.path.name, 'one', '2000-01-01T00:00:00.000001+00:00'],
+        ['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
+    ]
+
+    # Delete all Tags so they are recreated.
+    test_session.query(tags.Tag).delete()
+    # Delete all TagFileEntry(s), import them again.
+    test_session.query(TagZimEntry).delete()
+    test_session.commit()
+
+    tags.import_tags_config()
+
+    tag_zim_entries = test_session.query(TagZimEntry) \
+        .order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry)
+    assert [(i.tag.name, i.zim.path, i.zim_entry) for i in tag_zim_entries] \
+           == [('Tag2', test_zim.path, 'one'), ('Tag1', test_zim.path, 'two')]
+
+    # Tags were recreated.
+    assert {i[0] for i in test_session.query(tags.Tag.id)} == {3, 4}
+
+
+def test_zim_all_entries(test_session, test_zim):
+    entries = [i.__json__() for i in test_zim.all_entries]
+    assert len(entries) == 12
+
+
+@pytest.mark.asyncio
+async def test_zim_download(test_session, kiwix_download_zim, test_directory, test_zim_bytes, flags_lock):
+    await lib.subscribe('Wikipedia (with images)', 'es')
+
+    # Downloading the catalog should lead to a new Zim file being downloaded.
+    await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2023-06.zim')
+    recurring_download, once_download = test_session.query(Download).order_by(Download.id).all()
+    recurring_download: Download
+    once_download: Download
+    assert recurring_download.attempts == 1 and recurring_download.status == 'complete', \
+        'Recurring download should have succeeded'
+    assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
+    assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').is_file()
+    assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').stat().st_size
+    assert once_download.attempts == 1
+
+    # Download catalog again, should not re-download the existing file.
+    recurring_download.status = 'new'
+    test_session.commit()
+    await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2023-06.zim',
+                             download_file_side_effect=Exception('should not be called twice'))
+    recurring_download, once_download = test_session.query(Download).order_by(Download.id).all()
+    assert recurring_download.attempts == 2 and recurring_download.status == 'complete', \
+        'Recurring download should have happened again'
+    assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
+    # New file was modeled into a FileGroup.
+    assert test_session.query(FileGroup).count() == 1
+    # No outdated files exist.
+    assert not flags.outdated_zims.is_set()
+
+    # Download catalog again, should fetch the latest file.
+    recurring_download.status = 'new'
+    test_session.commit()
+    await kiwix_download_zim(expected_url='https://download.kiwix.org/zim/wikipedia/wikipedia_es_all_maxi_2024-01.zim',
+                             download_info=DownloadFileInfo(
+                                 status=HTTPStatus.OK,
+                                 name='wikipedia_es_all_maxi_2024-01.zim',
+                                 size=len(test_zim_bytes),
+                             ),
+                             hrefs=['?C=N;O=D', '?C=M;O=A', '?C=S;O=A', '?C=D;O=A', '/zim/',
+                                    'wikipedia_es_all_maxi_2023-05.zim',
+                                    'wikipedia_es_all_maxi_2023-06.zim',
+                                    'wikipedia_es_all_maxi_2024-01.zim',  # The new Zim.
+                                    'wikipedia_es_all_nopic_2023-05.zim',
+                                    'wikipedia_es_all_nopic_2023-06.zim', ]
+                             )
+    recurring_download, first_download, second_download = test_session.query(Download).order_by(Download.id).all()
+    assert recurring_download.attempts == 3 and recurring_download.status == 'complete', \
+        'Recurring download should have happened again'
+    assert not any([i[0] for i in test_session.query(Download.error)]), 'All downloads should have succeeded'
+    assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').is_file()
+    assert (test_directory / 'zims/wikipedia_es_all_maxi_2023-06.zim').stat().st_size
+    assert (test_directory / 'zims/wikipedia_es_all_maxi_2024-01.zim').is_file()
+    assert (test_directory / 'zims/wikipedia_es_all_maxi_2024-01.zim').stat().st_size
+    # New file was modeled.
+    assert test_session.query(FileGroup).count() == 2
+    # But old file is now outdated.
+    assert flags.outdated_zims.is_set()
+
+    # Delete the outdated file.
+    await lib.remove_outdated_zim_files()
+    assert test_session.query(FileGroup).count() == 1
+    # No longer outdated
+    assert not flags.outdated_zims.is_set()
+
+
+def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_factory, test_tags_config):
+    """When an outdated Zim file is deleted, the Tags should be moved to the latest Zim."""
+    zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim')  # The outdated Zim.
+    zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim')
+    zim3 = zim_factory('wikipedia_en_all_maxi_2020-03.zim')  # The latest Zim.
+    tag1, tag2 = tag_factory('tag1'), tag_factory('tag2')
+    # Tag outdated Zim entries.
+    zim1.tag_entry(tag1.name, 'home')
+    zim1.tag_entry(tag2.name, 'one')
+    # This should not be migrated.
+    zim2.tag_entry(tag2.name, 'two')
+    # Tag the latest Zim as well.
+    zim3.tag_entry(tag2.name, 'one')
+    test_session.commit()
+    assert test_session.query(TagZimEntry).count() == 4
+
+    # All Zims have been tagged.
+    tags_config = tags.get_tags_config()
+    tags_config.save_tags(session=test_session)
+    assert 'wikipedia_en_all_maxi_2020-01.zim' in (config_text := tags_config.get_file().read_text())
+    assert 'wikipedia_en_all_maxi_2020-02.zim' in config_text
+    assert 'wikipedia_en_all_maxi_2020-03.zim' in config_text
+
+    # Zim was deleted, as well as it's ZimTagEntry(s).
+    zim1.delete()
+    tags.import_tags_config(test_session)
+
+    tag_zim_entries = sorted([(i.tag.name, i.zim.path, i.zim_entry) for i in test_session.query(TagZimEntry)])
+    assert tag_zim_entries == [
+        ('tag1', zim3.path, 'home'),
+        ('tag2', zim2.path, 'two'),
+        ('tag2', zim3.path, 'one'),  # Both "one" tags were combined.
+    ]
+
+
+@pytest.mark.parametrize(
+    'url,expected',
+    [
+        ('https://download.kiwix.org/zim/wikipedia/wikipedia_arz_all_maxi_', ('Wikipedia (with images)', 'arz')),
+        ('https://download.kiwix.org/zim/wikipedia/wikipedia_arz_all_nopic_', ('Wikipedia (no images)', 'arz')),
+        ('https://download.kiwix.org/zim/wikipedia/wikipedia_ar_all_mini_', ('Wikipedia (mini)', 'ar')),
+        ('https://download.kiwix.org/zim/wikiversity/wikiversity_zh_all_maxi_', ('Wikiversity (with images)', 'zh')),
+        ('https://download.kiwix.org/zim/wikiversity/wikiversity_zh_all_nopic_', ('Wikiversity (no images)', 'zh')),
+        ('https://download.kiwix.org/zim/wiktionary/wiktionary_eo_all_maxi_', ('Wikitionary (with images)', 'eo')),
+        ('https://download.kiwix.org/zim/wiktionary/wiktionary_eo_all_nopic_', ('Wikitionary (no images)', 'eo')),
+        ('https://download.kiwix.org/zim/wikibooks/wikibooks_es_all_nopic_', ('Wikibooks (no images)', 'es')),
+        ('https://download.kiwix.org/zim/wikibooks/wikibooks_es_all_maxi_', ('Wikibooks (with images)', 'es')),
+        ('https://download.kiwix.org/zim/wikisource/wikisource_bn_all_maxi_', ('Wikisource', 'bn')),
+        ('https://download.kiwix.org/zim/vikidia/vikidia_ru_all_maxi_', ('Vikidia', 'ru')),
+        ('https://download.kiwix.org/zim/stack_exchange/raspberrypi.stackexchange.com_en_all_',
+         ('Raspberry Pi (Stack Exchange)', 'en')),
+        ('https://download.kiwix.org/zim/stack_exchange/stackoverflow.com_en_all_',
+         ('Stackoverflow (Stack Exchange)', 'en')),
+        ('https://download.kiwix.org/zim/stack_exchange/ja.stackoverflow.com_ja_all_',
+         ('Stackoverflow (Stack Exchange)', 'ja')),
+        ('https://download.kiwix.org/zim/stack_exchange/superuser.com_en_all_', ('Superuser (Stack Exchange)', 'en')),
+        ('https://download.kiwix.org/zim/ifixit/ifixit_nl_all_', ('iFixit', 'nl')),
+        ('https://download.kiwix.org/zim/gutenberg/gutenberg_ang_all_', ('Gutenberg', 'ang')),
+    ]
+)
+def test_zim_download_url_to_name(url, expected):
+    assert lib.zim_download_url_to_name(url) == expected
+
+
+@pytest.mark.asyncio
+async def test_zim_subscription_download_import(test_session, test_downloader_config):
+    # Subscription creates a ZimSubscription and Download
+    await lib.subscribe('Wikisource', 'en', session=test_session)
+    # Add a once-download.  This should not be associated with a ZimSubscription.
+    test_session.add(
+        Download(url='https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2021-03.zim', status='complete'))
+    test_session.commit()
+    download = test_session.query(Download).filter(Download.frequency.isnot(None)).one()
+    assert download.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
+
+    # Save config file, delete all entries.
+    await save_downloads_config()
+    test_session.query(Download).delete()
+    test_session.query(ZimSubscription).delete()
+    test_session.commit()
+
+    for _ in range(1):  # Import twice.
+        # Importing the config restores the Download and ZimSubscription.
+        await import_downloads_config(session=test_session)
+        recurring, once = test_session.query(Download).order_by(Download.frequency)
+        assert once.url == 'https://download.kiwix.org/zim/wikibooks/wikibooks_en_all_maxi_2021-03.zim'
+        assert test_session.query(Download).count() == 2
+        assert recurring.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
+        assert recurring.frequency
+        subscription: ZimSubscription = test_session.query(ZimSubscription).one()
+        assert subscription.download_id == recurring.id
+        assert subscription.name == 'Wikisource'
+        assert subscription.language == 'en'

nginx.conf

@@ -40,8 +40,16 @@ http {
       alias /opt/wrolpi/icon.png;
     }
 
-    location /epub.html {
-      alias /opt/wrolpi/app/public/epub.html;
+    location /epub/epub.html {
+      alias /opt/wrolpi/app/public/epub/epub.html;
+    }
+
+    location /epub/epub.js {
+      alias /opt/wrolpi/app/public/epub/epub.js;
+    }
+
+    location /epub/jszip.min.js {
+      alias /opt/wrolpi/app/public/jszip.min.js;
     }
 
     location / {

requirements.txt

@@ -9,6 +9,7 @@ beautifulsoup4==4.12.2
 cachetools==5.3.0
 ebooklib==0.18
 feedparser==6.0.10
+libzim==3.1.0
 mock==5.0.2
 psutil==5.9.5
 psycopg2-binary==2.9.6

scripts/dev_build.sh

@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+# Pulls, builds, and initializes the development Docker containers.
+
+docker volume create --name=openstreetmap-data
+docker volume create --name=openstreetmap-rendered-tiles
+docker-compose pull
+docker-compose build --parallel
+docker-compose up -d db
+docker-compose run --rm api db upgrade
+docker-compose stop

scripts/install_debian_11.sh

@@ -10,6 +10,7 @@ set -e
 apt install -y \
   apt-transport-https \
   ca-certificates \
+  calibre \
   chromium \
   chromium-driver \
   cpufrequtils \
@@ -19,6 +20,8 @@ apt install -y \
   gnupg-agent \
   hostapd \
   htop \
+  kiwix \
+  kiwix-tools \
   libpq-dev \
   network-manager \
   nginx-doc \
@@ -30,7 +33,8 @@ apt install -y \
   python3-pip \
   python3-venv \
   software-properties-common \
-  vim
+  vim \
+  zim-tools
 
 # Install npm.
 npm --version || curl -fsSL https://deb.nodesource.com/setup_18.x | bash -

scripts/install_raspberrypios.sh

@@ -10,6 +10,7 @@ set -e
 apt install -y \
   apt-transport-https \
   ca-certificates \
+  calibre \
   chromium \
   chromium-driver \
   cpufrequtils \
@@ -19,6 +20,8 @@ apt install -y \
   gnupg-agent \
   hostapd \
   htop \
+  kiwix \
+  kiwix-tools \
   libpq-dev \
   network-manager \
   nginx-doc \
@@ -30,7 +33,8 @@ apt install -y \
   python3-pip \
   python3-venv \
   software-properties-common \
-  vim
+  vim \
+  zim-tools
 
 # Install npm.
 npm --version || curl -fsSL https://deb.nodesource.com/setup_18.x | bash -

wrolpi/common.py

@@ -2,6 +2,7 @@ import asyncio
 import atexit
 import contextlib
 import inspect
+import json
 import logging
 import multiprocessing
 import os
@@ -12,9 +13,11 @@ import sys
 import tempfile
 from asyncio import Task
 from copy import deepcopy
+from dataclasses import dataclass
 from datetime import datetime, date
 from decimal import Decimal
 from functools import wraps
+from http import HTTPStatus
 from itertools import islice, filterfalse, tee
 from multiprocessing import Lock, Manager
 from pathlib import Path
@@ -24,15 +27,16 @@ from urllib.parse import urlparse, urlunsplit
 
 import aiohttp
 import yaml
+from aiohttp import ClientResponse
+from bs4 import BeautifulSoup
 from sqlalchemy import types
 from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import Session
 
-from wrolpi.dates import now, from_timestamp
-from wrolpi.errors import WROLModeEnabled, NativeOnly
-from wrolpi.vars import PYTEST, MODULES_DIR, \
-    DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY
+from wrolpi.dates import now, from_timestamp, seconds_to_timestamp
+from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError
+from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY
 
 logger = logging.getLogger()
 ch = logging.StreamHandler()
@@ -68,62 +72,69 @@ def log_level_context(level):
 
 
 __all__ = [
-    'logger',
-    'set_log_level',
     'Base',
-    'ModelHelper',
-    'get_model_by_table_name',
-    'tsvector',
-    'compile_tsvector',
-    'WROLPI_CONFIG',
-    'sanitize_link',
     'ConfigFile',
-    'get_config',
-    'set_test_config',
-    'wrol_mode_enabled',
-    'wrol_mode_check',
-    'enable_wrol_mode',
-    'disable_wrol_mode',
-    'insert_parameter',
-    'iterify',
-    'date_range',
-    'remove_whitespace',
-    'run_after',
-    'set_test_media_directory',
-    'get_media_directory',
-    'check_media_directory',
-    'get_absolute_media_path',
-    'get_relative_to_media_directory',
-    'get_files_and_directories',
-    'minimize_dict',
-    'make_media_directory',
-    'extract_domain',
-    'import_modules',
-    'api_param_limiter',
-    'partition',
-    'chdir',
-    'zig_zag',
-    'walk',
-    'escape_file_name',
-    'native_only',
-    'recursive_map',
+    'DownloadFileInfo',
+    'DownloadFileInfoLink',
+    'ModelHelper',
+    'WROLPI_CONFIG',
+    'aiohttp_get',
+    'aiohttp_head',
     'aiohttp_post',
-    'register_modeler',
+    'api_param_limiter',
     'apply_modelers',
-    'register_refresh_cleanup',
     'apply_refresh_cleanup',
-    'match_paths_to_suffixes',
-    'chunks',
-    'chunks_by_stem',
-    'timer',
-    'cum_timer',
-    'limit_concurrent',
-    'truncate_object_bytes',
     'background_task',
-    'get_warn_once',
-    'truncate_generator_bytes',
     'cancel_refresh_tasks',
     'cancelable_wrapper',
+    'chdir',
+    'check_media_directory',
+    'chunks',
+    'chunks_by_stem',
+    'compile_tsvector',
+    'cum_timer',
+    'date_range',
+    'disable_wrol_mode',
+    'download_file',
+    'enable_wrol_mode',
+    'escape_file_name',
+    'extract_domain',
+    'extract_headlines',
+    'extract_html_text',
+    'get_absolute_media_path',
+    'get_config',
+    'get_download_info',
+    'get_files_and_directories',
+    'get_media_directory',
+    'get_model_by_table_name',
+    'get_relative_to_media_directory',
+    'get_warn_once',
+    'insert_parameter',
+    'iterify',
+    'limit_concurrent',
+    'logger',
+    'make_media_directory',
+    'match_paths_to_suffixes',
+    'minimize_dict',
+    'native_only',
+    'partition',
+    'recursive_map',
+    'register_modeler',
+    'register_refresh_cleanup',
+    'remove_whitespace',
+    'run_after',
+    'sanitize_link',
+    'set_log_level',
+    'set_test_config',
+    'set_test_media_directory',
+    'timer',
+    'truncate_generator_bytes',
+    'truncate_object_bytes',
+    'tsvector',
+    'walk',
+    'wrol_mode_check',
+    'wrol_mode_enabled',
+    'zig_zag',
 ]
 
 # Base is used for all SQLAlchemy models.
@@ -278,6 +289,7 @@ class WROLPiConfig(ConfigFile):
         hotspot_on_startup=True,
         hotspot_password='wrolpi hotspot',
         hotspot_ssid='WROLPi',
+        ignore_outdated_zims=False,
         throttle_on_startup=False,
         wrol_mode=False,
     )
@@ -330,6 +342,14 @@ class WROLPiConfig(ConfigFile):
     def hotspot_ssid(self, value: str):
         self.update({'hotspot_ssid': value})
 
+    @property
+    def ignore_outdated_zims(self) -> bool:
+        return self._config['ignore_outdated_zims']
+
+    @ignore_outdated_zims.setter
+    def ignore_outdated_zims(self, value: bool):
+        self.update({'ignore_outdated_zims': value})
+
     @property
     def throttle_on_startup(self) -> bool:
         return self._config['throttle_on_startup']
@@ -626,22 +646,6 @@ def extract_domain(url):
     return domain
 
 
-def import_modules():
-    """Import all WROLPi Modules in the modules directory.  Raise an ImportError if there are no modules."""
-    modules = [i.name for i in MODULES_DIR.iterdir() if
-               i.is_dir() and not (i.name.startswith('_') or i.name.startswith('.'))]
-    imported = []
-    for module_name in modules:
-        module = f'modules.{module_name}.api'
-        logger_.debug(f'Importing {module}')
-        try:
-            __import__(module, globals(), locals(), [], 0)
-            imported.append(module_name)
-        except ImportError as e:
-            logger_.fatal(f'Unable to import {module}', exc_info=e)
-    return imported
-
-
 def api_param_limiter(maximum: int, default: int = 20) -> callable:
     """Create a function which restricts the maximum number that can be returned.
     Useful for restricting API limit params.
@@ -817,14 +821,233 @@ def recursive_map(obj: Any, func: callable):
     return func(obj)
 
 
-async def aiohttp_post(url: str, json_, timeout: int = None) -> Tuple[Dict, int]:
-    """Perform an async aiohttp post request.  Return the json contents."""
+@contextlib.asynccontextmanager
+async def aiohttp_session(timeout: int = None):
+    """Convenience function because aiohttp timeout cannot be None."""
+    if timeout:
         timeout = aiohttp.ClientTimeout(total=timeout) if timeout else None
         async with aiohttp.ClientSession(timeout=timeout) as session:
+            yield session
+    else:
+        async with aiohttp.ClientSession() as session:
+            yield session
+
+
+async def aiohttp_post(url: str, json_, timeout: int = None) -> Tuple[Dict, int]:
+    """Perform an async aiohttp POST request.  Return the json contents."""
+    async with aiohttp_session(timeout) as session:
         async with session.post(url, json=json_) as response:
             return await response.json(), response.status
 
 
+async def aiohttp_get(url: str, timeout: int = None, headers: dict = None) -> Tuple[bytes, int]:
+    """Perform an async aiohttp GET request.  Return the contents."""
+    async with aiohttp_session(timeout) as session:
+        async with session.get(url, headers=headers) as response:
+            return await response.content.read(), response.status
+
+
+async def aiohttp_head(url: str, timeout: int = None) -> Tuple[ClientResponse, int]:
+    """Perform an async aiohttp HEAD request.  Return the contents."""
+    async with aiohttp_session(timeout) as session:
+        async with session.head(url) as response:
+            return response, response.status
+
+
+async def speed_test(url: str) -> int:
+    """Request the last megabyte of the provided url, return the content length divided by the time elapsed (speed)."""
+    response, status = await aiohttp_head(url)
+    content_length = response.headers['Content-Length']
+    start_bytes = int(content_length) - 10485760
+    range_ = f'bytes={start_bytes}-{content_length}'
+    start_time = datetime.now()
+    content, status = await aiohttp_get(url, headers={'Range': range_})
+    elapsed = int((datetime.now() - start_time).total_seconds())
+    return len(content) // elapsed
+
+
+async def get_fastest_mirror(urls: List[str]) -> str:
+    """Perform a speed test on each URL, return the fastest."""
+    fastest_url = urls[0]
+    fastest_speed = 0
+    for url in urls:
+        try:
+            speed = await speed_test(url)
+        except Exception as e:
+            logger.error(f'Speedtest of {url} failed', exc_info=e)
+            continue
+
+        if speed > fastest_speed:
+            fastest_url = url
+
+    return fastest_url
+
+
+@dataclass
+class DownloadFileInfoLink:
+    """Represents an HTTP "Link" Header."""
+    url: str
+    rel: str
+    type: str
+    priority: int
+    geo: str
+
+
+@dataclass
+class DownloadFileInfo:
+    """Information about a file that can be downloaded."""
+    name: str = None
+    size: int = None
+    type: str = None
+    accept_ranges: str = None
+    status: int = None
+    location: str = None
+    links: List[DownloadFileInfoLink] = None
+
+
+FILENAME_MATCHER = re.compile(r'.*filename="(.*)"')
+
+
+async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo:
+    """Gets information (name, size, etc.) about a downloadable file at the provided URL."""
+    response, status = await aiohttp_head(url, timeout)
+    try:
+        links = response.headers.getall('Link')
+    except KeyError:
+        links = None
+
+    new_links = list()
+    if links:
+        # Convert "Link" header strings to DownloadFileInfoLink.
+        for idx, link in enumerate(links):
+            url, *props = link.split(';')
+            url = url[1:-1]
+            properties = dict()
+            for prop in props:
+                name, value = prop.strip().split('=')
+                properties[name] = value
+            new_links.append(DownloadFileInfoLink(
+                url,
+                properties.get('rel').strip() if 'rel' in properties else None,
+                properties.get('type').strip() if 'type' in properties else None,
+                int(properties.get('pri').strip()) if 'pri' in properties else None,
+                properties.get('geo').strip() if 'geo' in properties else None,
+            ))
+
+    info = DownloadFileInfo(
+        type=response.headers.get('Content-Type'),
+        size=int(response.headers['Content-Length']) if 'Content-Length' in response.headers else None,
+        accept_ranges=response.headers.get('Accept-Ranges'),
+        status=response.status,
+        location=response.headers.get('Location'),
+        links=new_links,
+    )
+
+    disposition = response.headers.get('Content-Disposition')
+
+    if disposition and 'filename' in disposition:
+        if (match := FILENAME_MATCHER.match(disposition)) and (groups := match.groups()):
+            info.name = groups[0]
+    else:
+        # No Content-Disposition with filename, use the URL name.
+        parsed = urlparse(url)
+        info.name = parsed.path.split('/')[-1]
+
+    return info
+
+
+download_logger = logger_.getChild('download')
+
+
+async def download_file(url: str, output_path: pathlib.Path = None, info: DownloadFileInfo = None,
+                        timeout: int = 7 * 24 * 60 * 60):
+    """Uses aiohttp to download an HTTP file.  Performs a speed test when mirrors are found, downloads from the fastest
+    mirror.
+
+    Attempts to resume the file if `output_path` already exists.
+
+    @warning: Timeout default is a week because of large downloads.
+    """
+    info = info or await get_download_info(url, timeout)
+
+    if output_path.is_file() and info.size == output_path.stat().st_size:
+        download_logger.warning(f'Already downloaded {repr(str(url))} to {repr(str(output_path))}')
+        return
+
+    if info.links and (mirror_urls := [i.url for i in info.links if i.rel == 'duplicate']):
+        # Mirrors are available, find the fastest.
+        download_logger.info(f'Performing download speed test on mirrors: {mirror_urls}')
+        url = await get_fastest_mirror(mirror_urls)
+        info = await get_download_info(url, timeout)
+
+    total_size = info.size
+
+    download_logger.info(f'Starting download of {url} with {total_size} total bytes')
+    if info.accept_ranges == 'bytes' or not output_path.is_file():
+        with output_path.open('ab') as fh:
+            headers = dict()
+            # Check the position of append, if it is 0 then we do not need to resume.
+            position = fh.tell()
+            if position:
+                headers['Range'] = f'bytes={position}-'
+
+            async with aiohttp_session(timeout=timeout) as session:
+                async with session.get(url, headers=headers) as response:
+                    response: ClientResponse
+                    if response.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:
+                        download_logger.warning(f'Server responded with 416, file is probably already downloaded')
+                        return
+
+                    if position and response.status != HTTPStatus.PARTIAL_CONTENT:
+                        raise UnrecoverableDownloadError(
+                            f'Tried to resume {repr(str(url))} but got status {response.status}')
+
+                    # May or may not be using Range.  Append each chunk to the output file.
+                    last_report = datetime.now()
+                    bytes_received = 0
+                    async for data in response.content.iter_any():
+                        fh.write(data)
+
+                        bytes_received += len(data)
+                        if (elapsed := (datetime.now() - last_report).total_seconds()) > 10:
+                            # Report download speed every 10 seconds.
+                            bytes_per_second = int(bytes_received // elapsed)
+                            download_logger.debug(f'{bytes_received=} {elapsed=} {bytes_per_second=}')
+                            size = fh.tell()
+                            bytes_remaining = total_size - size
+                            seconds_remaining = bytes_remaining // bytes_per_second
+                            percent = int((size / total_size) * 100)
+                            download_logger.info(
+                                f'Downloading {url} at'
+                                f' rate={human_bandwidth(bytes_per_second)}'
+                                f' estimate={seconds_to_timestamp(seconds_remaining)}')
+                            download_logger.debug(f'Downloading {url} {total_size=} {size=} {percent=}')
+                            last_report = datetime.now()
+                            bytes_received = 0
+
+                        # TODO this cannot be canceled.
+                        # Sleep to catch cancel.
+                        await asyncio.sleep(0)
+    elif output_path.is_file():
+        # TODO support downloading files that cannot be resumed.
+        raise UnrecoverableDownloadError(f'Cannot resume download {url}')
+
+
+def human_bandwidth(bps: int) -> str:
+    """Convert bits per second to a more readable format.
+
+    >>> human_bandwidth(2000)
+    # '2 Kbps'
+    """
+    units = ['bps', 'Kbps', 'Mbps', 'Gbps', 'Tbps']
+    unit = 0
+    while bps > 1000 and unit <= len(units):
+        bps /= 1000
+        unit += 1
+
+    return f'{int(bps)} {units[unit]}'
+
+
 modelers = []
 
 
@@ -1189,3 +1412,48 @@ def url_strip_host(url: str) -> str:
     url = urlparse(url)
     url = urlunsplit(('', '', url.path, url.query, url.fragment))
     return url or '/'
+
+
+def extract_html_text(html: str) -> str:
+    soup = BeautifulSoup(html, features='html.parser')
+
+    # kill all script and style elements
+    for script in soup(["script", "style"]):
+        script.extract()  # rip it out
+
+    text = soup.body.get_text()
+
+    # break into lines and remove leading and trailing space on each
+    lines = (line.strip() for line in text.splitlines())
+    # break multi-headlines into a line each
+    chunks_ = (phrase.strip() for line in lines for phrase in line.split("  "))
+    # drop blank lines
+    text = '\n'.join(chunk for chunk in chunks_ if chunk)
+
+    return text
+
+
+def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, float]]:
+    """Use Postgres to extract Headlines and ranks of the `search_str` from the provided entries."""
+    from wrolpi.db import get_db_curs
+
+    source = json.dumps([{'content': i} for i in entries])
+    with get_db_curs() as curs:
+        stmt = '''
+        WITH vectored AS (
+            -- Convert the "source" json to a recordset.
+            with source as (select * from json_to_recordset(%s::json) AS (content TEXT))
+            select
+                to_tsvector('english'::regconfig, source.content) AS vector,
+                source.content
+            from source
+        )
+        SELECT
+            ts_headline(vectored.content, websearch_to_tsquery(%s), 'MaxFragments=10, MaxWords=8, MinWords=7'),
+            ts_rank(vectored.vector, websearch_to_tsquery(%s))
+        FROM vectored
+        '''
+        curs.execute(stmt, [source, search_str, search_str])
+        headlines = [tuple(i) for i in curs.fetchall()]
+
+    return headlines

wrolpi/conftest.py

@@ -11,6 +11,7 @@ import shutil
 import tempfile
 import zipfile
 from abc import ABC
+from datetime import datetime
 from itertools import zip_longest
 from typing import List, Callable, Union, Dict, Sequence
 from typing import Tuple, Set
@@ -22,7 +23,8 @@ import pytest
 import sqlalchemy
 import yaml
 from PIL import Image
-from sanic_testing.testing import SanicTestClient
+from sanic_testing.reusable import ReusableClient
+from sanic_testing.testing import SanicASGITestClient
 from sqlalchemy.engine import Engine, create_engine
 from sqlalchemy.orm import Session, sessionmaker
 
@@ -120,10 +122,11 @@ def test_config(test_directory) -> pathlib.Path:
 ROUTES_ATTACHED = False
 
 
-@pytest.fixture(autouse=True)
-def test_client() -> SanicTestClient:
-    """
-    Get a Sanic Test Client with all default routes attached.
+@pytest.fixture()
+def test_client() -> ReusableClient:
+    """Get a Reusable Sanic Test Client with all default routes attached.
+
+    (A non-reusable client would turn on for each request)
     """
     global ROUTES_ATTACHED
     if ROUTES_ATTACHED is False:
@@ -132,7 +135,22 @@ def test_client() -> SanicTestClient:
             api_app.blueprint(bp)
         ROUTES_ATTACHED = True
 
-    yield api_app.test_client
+    client = ReusableClient(api_app)
+    with client:
+        yield client
+
+
+@pytest.fixture()
+def test_async_client() -> SanicASGITestClient:
+    """Get an Async Sanic Test Client with all default routes attached."""
+    global ROUTES_ATTACHED
+    if ROUTES_ATTACHED is False:
+        # Attach any blueprints for the test.
+        for bp in BLUEPRINTS:
+            api_app.blueprint(bp)
+        ROUTES_ATTACHED = True
+
+    yield SanicASGITestClient(api_app)
 
 
 @pytest.fixture
@@ -163,6 +181,7 @@ async def test_download_manager(
 @pytest.fixture
 def fake_now():
     try:
+        set_test_now(datetime(2000, 1, 1))
         yield set_test_now
     finally:
         # reset now() to its original functionality.
@@ -279,6 +298,11 @@ def vtt_file2(test_directory) -> pathlib.Path:
     yield destination
 
 
+@pytest.fixture
+def srt_text() -> str:
+    return (PROJECT_DIR / 'test/example3.en.srt').read_text()
+
+
 @pytest.fixture
 def srt_file3(test_directory) -> pathlib.Path:
     """Return a copy of the example3 SRT file in the `test_directory`."""
@@ -407,10 +431,13 @@ def tag_factory(test_session):
     names = ['one', 'two', 'three', 'four', 'five', 'six']
     count = 1
 
-    def factory() -> Tag:
-        tag = Tag(name=names.pop(0), color=f'#{str(count) * 6}')
+    def factory(name: str = None) -> Tag:
+        if not name:
+            name = names.pop(0)
+        tag = Tag(name=name, color=f'#{str(count) * 6}')
         test_session.add(tag)
         test_session.commit()
+        test_session.flush([tag, ])
         return tag
 
     return factory

wrolpi/db.py

@@ -1,4 +1,4 @@
-import pathlib
+import inspect
 import types
 from contextlib import contextmanager
 from functools import wraps
@@ -6,9 +6,9 @@ from typing import ContextManager, Tuple, List, Union, Type, Generator
 
 import psycopg2
 import psycopg2.extensions
+import sqlalchemy
 import sqlalchemy.exc
 from psycopg2._psycopg import cursor
-from sqlalchemy import create_engine, inspect
 from sqlalchemy.orm import sessionmaker, Session
 from sqlalchemy.pool import NullPool
 
@@ -40,14 +40,15 @@ def get_db_args(dbname: str = None):
 # This engine is used to modify the databases.
 connect_args = dict(application_name='wrolpi_api_super')
 postgres_args = get_db_args('postgres')
-postgres_engine = create_engine('postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**postgres_args),
+postgres_engine = sqlalchemy.create_engine(
+    'postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**postgres_args),
     execution_options={'isolation_level': 'AUTOCOMMIT'}, connect_args=connect_args)
 
 # This engine is used for all normal tasks (except testing).
 db_args = get_db_args()
 connect_args = dict(application_name='wrolpi_api', connect_timeout=1)
 uri = 'postgresql://{user}:{password}@{host}:{port}/{dbname}'.format(**db_args)
-engine = create_engine(uri, poolclass=NullPool, connect_args=connect_args)
+engine = sqlalchemy.create_engine(uri, poolclass=NullPool, connect_args=connect_args)
 session_maker = sessionmaker(bind=engine)
 
 LOGGED_ARGS = False
@@ -69,7 +70,7 @@ def _get_db_session():
     return engine, session
 
 
-def get_db_context() -> Tuple[create_engine, Session]:
+def get_db_context() -> Tuple[sqlalchemy.create_engine, Session]:
     """
     Get a DB engine and session.
     """
@@ -122,7 +123,7 @@ def get_db_curs(commit: bool = False):
             connection.rollback()
 
 
-def optional_session(commit: Union[callable, bool] = False):
+def optional_session(commit: Union[callable, bool] = False) -> callable:
     """
     Wraps a function, if a Session is passed it will be used.  Otherwise, a new session will be
     created and passed to the function.
@@ -140,15 +141,26 @@ def optional_session(commit: Union[callable, bool] = False):
 
     def call_func(func, session, args, kwargs):
         if session:
-            return func(*args, session=session, **kwargs)
+            result = func(*args, session=session, **kwargs)
+            if commit:
+                session.commit()
         else:
             with get_db_session() as session:
-                return func(*args, session=session, **kwargs)
+                result = func(*args, session=session, **kwargs)
+                if commit:
+                    session.commit()
+        return result
 
     def wrapper(*w_args, **w_kwargs):
         if len(w_args) == 1 and len(w_kwargs) == 0 and callable(w_args[0]):
             func = w_args[0]
 
+            if inspect.iscoroutinefunction(func):
+                @wraps(func)
+                async def wrapped(*args, session: Session = None, **kwargs):
+                    session, args, kwargs = find_session(*args, session=session, **kwargs)
+                    return await call_func(func, session, args, kwargs)
+            else:
                 @wraps(func)
                 def wrapped(*args, session: Session = None, **kwargs):
                     session, args, kwargs = find_session(*args, session=session, **kwargs)
@@ -156,6 +168,7 @@ def optional_session(commit: Union[callable, bool] = False):
 
             return wrapped
         else:
+            # No params were passed to this wrapper, `commit` is actually the function we are wrapping.
             session_, w_args, w_kwargs = find_session(*w_args, **w_kwargs)
             return call_func(commit, session_, w_args, w_kwargs)
 
@@ -165,7 +178,7 @@ def optional_session(commit: Union[callable, bool] = False):
 @optional_session
 def get_ranked_models(ranked_primary_keys: List, model: Type[Base], session: Session = None) -> List[Base]:
     """Get all objects whose primary keys are in the `ranked_primary_keys`, preserve their order."""
-    pkey = inspect(model).primary_key[0]
+    pkey = sqlalchemy.inspect(model).primary_key[0]
     pkey_name = pkey.name
     results = list(session.query(model).filter(pkey.in_(ranked_primary_keys)).all())
     results = sorted(results, key=lambda i: ranked_primary_keys.index(getattr(i, pkey_name)))

wrolpi/downloader.py

@@ -1,6 +1,5 @@
 import asyncio
 import contextlib
-import json
 import logging
 import multiprocessing
 import os
@@ -10,9 +9,7 @@ from abc import ABC
 from dataclasses import dataclass, field
 from datetime import timedelta, datetime
 from enum import Enum
-from functools import partial
 from itertools import filterfalse
-from operator import attrgetter
 from queue import Empty
 from typing import List, Dict, Generator, Iterable
 from typing import Tuple, Optional
@@ -45,6 +42,7 @@ class DownloadFrequency(int, Enum):
     biweekly = weekly * 2
     days30 = daily * 30
     days90 = daily * 90
+    days180 = daily * 180
 
 
 @dataclass
@@ -371,7 +369,6 @@ class DownloadManager:
                     result = DownloadResult(success=False, error=str(traceback.format_exc()))
                     try_again = False
                 except Exception as e:
-
                     worker_logger.warning(f'Failed to download {url}.  Will be tried again later.', exc_info=e)
                     result = DownloadResult(success=False, error=str(traceback.format_exc()))
 
@@ -503,7 +500,7 @@ class DownloadManager:
             return downloader
         raise InvalidDownload(f'Cannot find downloader with name {name}')
 
-    def get_or_create_download(self, url: str, session: Session) -> Download:
+    def get_or_create_download(self, url: str, session: Session, reset_attempts: bool = False) -> Download:
         """Get a Download by its URL, if it cannot be found create one."""
         if not url:
             raise ValueError('Download must have a URL')
@@ -511,6 +508,9 @@ class DownloadManager:
         download = self.get_download(session, url=url)
         if not download:
             if url in get_download_manager_config().skip_urls:
+                if reset_attempts is True:
+                    self.remove_from_skip_list(url)
+                else:
                     raise InvalidDownload(
                         f'Refusing to download {url} because it is in the download_manager.yaml skip list')
             download = Download(url=url, status='new')
@@ -1086,6 +1086,9 @@ async def import_downloads_config(session: Session):
         logger.warning(f'Refusing to import downloads config when DB is not up.')
         return
 
+    from modules.zim.lib import zim_download_url_to_name
+    from modules.zim.models import ZimSubscription
+
     try:
         logger.warning('Importing downloads in config')
 
@@ -1124,6 +1127,29 @@ async def import_downloads_config(session: Session):
         logger.error('Failed to import downloads', exc_info=e)
         raise
 
+    try:
+        # Claim any Kiwix subscriptions for ZimSubscription(s).
+        downloads: List[Download] = session.query(Download)
+        need_commit = False
+        for download in downloads:
+            if not download.url.startswith('https://download.kiwix.org/') or not download.frequency:
+                # ZimSubscription requires download.kiwix.org AND a frequency.
+                continue
+
+            name, language = zim_download_url_to_name(download.url)
+            subscription = session.query(ZimSubscription).filter_by(name=name, language=language).one_or_none()
+            if not subscription:
+                subscription = ZimSubscription(name=name, language=language)
+            subscription.change_download(download.url, download.frequency, session=session)
+            session.add(subscription)
+            need_commit = True
+
+        if need_commit:
+            session.commit()
+    except Exception as e:
+        logger.error('Failed to restore ZimSubscriptions', exc_info=e)
+        raise
+
 
 def parse_feed(url: str) -> FeedParserDict:
     """Calls `feedparser.parse`, used for testing."""

wrolpi/errors.py

@@ -172,6 +172,30 @@ class FileUploadFailed(APIError):
     pass
 
 
+class UnknownZim(APIError):
+    pass
+
+
+class UnknownZimEntry(APIError):
+    pass
+
+
+class UnknownZimTagEntry(APIError):
+    pass
+
+
+class UnknownZimSubscription(APIError):
+    pass
+
+
+class FileConflict(APIError):
+    pass
+
+
+class DirectoryConflict(APIError):
+    pass
+
+
 error_codes = iter(range(1, 1000))
 
 API_ERRORS = {
@@ -380,4 +404,29 @@ API_ERRORS = {
         'message': 'Failed to upload file',
         'status': HTTPStatus.BAD_REQUEST,
     },
+    UnknownZim: {
+        'code': next(error_codes),
+        'message': 'Failed to find the Zim',
+        'status': HTTPStatus.NOT_FOUND,
+    },
+    UnknownZimEntry: {
+        'code': next(error_codes),
+        'message': 'Failed to find the Zim entry at that path',
+        'status': HTTPStatus.NOT_FOUND,
+    },
+    UnknownZimTagEntry: {
+        'code': next(error_codes),
+        'message': 'Failed to find the TagZimEntry',
+        'status': HTTPStatus.NOT_FOUND,
+    },
+    UnknownZimSubscription: {
+        'code': next(error_codes),
+        'message': 'Failed to find the ZimSubscription',
+        'status': HTTPStatus.NOT_FOUND,
+    },
+    FileConflict: {
+        'code': next(error_codes),
+        'message': 'File/directory already exists',
+        'status': HTTPStatus.CONFLICT,
+    },
 }

wrolpi/files/__init__.py

@@ -1,4 +1,4 @@
+from . import downloader  # noqa
 from . import ebooks  # noqa
 from . import pdfs  # noqa
 from .api import bp
-from . import downloader  # noqa

wrolpi/files/api.py

@@ -1,4 +1,5 @@
 import pathlib
+import shutil
 from http import HTTPStatus
 from multiprocessing import Manager
 from typing import List
@@ -8,11 +9,13 @@ from sanic import response, Request
 from sanic_ext import validate
 from sanic_ext.extensions.openapi import openapi
 
-from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory, logger, background_task
-from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed
+from wrolpi.common import get_media_directory, wrol_mode_check, get_relative_to_media_directory, logger, \
+    background_task, walk
+from wrolpi.errors import InvalidFile, UnknownDirectory, FileUploadFailed, FileConflict
 from wrolpi.root_api import get_blueprint, json_response
 from . import lib, schema
 from ..schema import JSONErrorResponse
+from ..vars import PYTEST
 
 bp = get_blueprint('Files', '/api/files')
 
@@ -50,9 +53,10 @@ async def get_file(_: Request, body: schema.FileRequest):
 )
 @validate(schema.DeleteRequest)
 async def delete_file(_: Request, body: schema.DeleteRequest):
-    if not body.file:
-        raise InvalidFile('file cannot be empty')
-    lib.delete_file(body.file)
+    paths = [i for i in body.paths if i]
+    if not paths:
+        raise InvalidFile('paths cannot be empty')
+    lib.delete(*paths)
     return response.empty()
 
 
@@ -149,6 +153,125 @@ async def post_search_directories(_, body: schema.DirectoriesSearchRequest):
     return json_response(body)
 
 
+@bp.post('/get_directory')
+@openapi.definition(
+    summary='Get data about a directory',
+    body=schema.Directory,
+)
+@validate(schema.Directory)
+async def post_get_directory(_: Request, body: schema.Directory):
+    path = get_media_directory() / body.path
+    size = 0
+    file_count = 0
+    directory_count = 0
+    for file in walk(path):
+        if file.is_file():
+            size += file.stat().st_size
+            file_count += 1
+        elif file.is_dir():
+            directory_count += 1
+    body = {
+        'path': path,
+        'size': size,
+        'file_count': file_count,
+        'directory_count': directory_count,
+    }
+    return json_response(body)
+
+
+@bp.post('/directory')
+@openapi.definition(
+    summary='Create a directory in the media directory.',
+    body=schema.Directory,
+)
+@validate(schema.Directory)
+async def post_create_directory(_: Request, body: schema.Directory):
+    path = get_media_directory() / body.path
+    try:
+        path.mkdir()
+    except FileExistsError:
+        raise FileConflict(f'{path} already exists')
+
+    if PYTEST:
+        await lib.refresh_files([path, ])
+    else:
+        background_task(lib.refresh_files([path, ]))
+
+    return response.empty(HTTPStatus.CREATED)
+
+
+@bp.post('/move')
+@openapi.definition(
+    summary='Move a file/directory into another directory in the media directory.',
+    body=schema.Move,
+)
+@validate(schema.Move)
+async def post_move(_: Request, body: schema.Move):
+    destination = get_media_directory() / body.destination
+    if not destination.is_dir():
+        raise UnknownDirectory(f'Cannot move files into {destination} because it does not exist')
+    sources = [get_media_directory() / i for i in body.paths]
+    for source in sources:
+        if not source.exists():
+            raise FileNotFoundError(f'Cannot find {source} to move')
+
+    try:
+        await lib.move(destination, *sources)
+    except Exception as e:
+        raise FileConflict(f'Failed to move {sources} to {destination}') from e
+
+    return response.empty(HTTPStatus.NO_CONTENT)
+
+
+@bp.post('/rename')
+@openapi.definition(
+    summary='Rename a file/directory in-place.',
+    body=schema.Rename,
+)
+@validate(schema.Rename)
+async def post_rename(_: Request, body: schema.Rename):
+    path = get_media_directory() / body.path
+    new_path = path.with_name(body.new_name)
+    if not path.exists():
+        raise FileNotFoundError(f'Cannot rename path {str(repr(path))} because it does not exist')
+    elif new_path.exists():
+        raise FileConflict(f'File already exists: {new_path}')
+
+    try:
+        await lib.rename(path, body.new_name)
+    except Exception as e:
+        logger.error(f'Failed to rename {path} to {new_path}', exc_info=e)
+        raise FileConflict(f'Failed to rename {path} to {new_path}') from e
+
+    if PYTEST:
+        await lib.refresh_files([path, new_path])
+    else:
+        background_task(lib.refresh_files([path, new_path]))
+
+    return response.empty(HTTPStatus.NO_CONTENT)
+
+
+@bp.post('/delete_directory')
+@openapi.definition(
+    summary='Delete a directory in the media directory.',
+    body=schema.Directory,
+)
+@validate(schema.Directory)
+async def delete_directory(_: Request, body: schema.Directory):
+    path = get_media_directory() / body.path
+    if path.is_dir():
+        shutil.rmtree(path)
+    else:
+        raise FileNotFoundError(f'No directory found: {path}')
+
+    if PYTEST:
+        await lib.refresh_files([path, ])
+    else:
+        background_task(lib.refresh_files([path, ]))
+
+    return response.empty()
+
+
 @bp.post('/tag')
 @validate(schema.TagFileGroupPost)
 async def post_tag_file_group(_, body: schema.TagFileGroupPost):

wrolpi/files/downloader.py

@@ -1,105 +1,21 @@
-import asyncio
 import pathlib
-import re
 import traceback
 from abc import ABC
-from dataclasses import dataclass
-from http import HTTPStatus
 from typing import List
-from urllib.parse import urlparse
 
-import aiohttp
-
-from wrolpi.common import get_media_directory, logger, background_task
+from wrolpi.common import get_media_directory, logger, background_task, get_download_info, \
+    download_file
 from wrolpi.db import get_db_session
 from wrolpi.downloader import Downloader, Download, DownloadResult
 from wrolpi.errors import UnrecoverableDownloadError
-
-__all__ = ['get_download_info', 'download_file', 'FileDownloader', 'file_downloader']
-
 from wrolpi.files.models import FileGroup
-
 from wrolpi.vars import PYTEST
 
+__all__ = ['FileDownloader', 'file_downloader']
+
 logger = logger.getChild(__name__)
 
 
-@dataclass
-class DownloadFileInfo:
-    name: str = None
-    size: int = None
-    type: str = None
-    accept_ranges: str = None
-
-
-FILENAME_MATCHER = re.compile(r'.*filename="(.*)"')
-
-
-async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo:
-    """Gets information (name, size, etc.) about a downloadable file at the provided URL."""
-    timeout = aiohttp.ClientTimeout(total=timeout) if timeout is not None else None
-    async with aiohttp.ClientSession(timeout=timeout) as session:
-        async with session.head(url) as response:
-            info = DownloadFileInfo(
-                type=response.headers.get('Content-Type'),
-                size=int(response.headers['Content-Length']) * 8 if 'Content-Length' in response.headers else None,
-                accept_ranges=response.headers.get('Accept-Ranges'),
-            )
-
-            disposition = response.headers.get('Content-Disposition')
-
-            if disposition and 'filename' in disposition:
-                if (match := FILENAME_MATCHER.match(disposition)) and (groups := match.groups()):
-                    info.name = groups[0]
-            else:
-                # No Content-Disposition with filename, use the URL name.
-                parsed = urlparse(url)
-                info.name = parsed.path.split('/')[-1]
-
-            return info
-
-
-async def download_file(url: str, info: DownloadFileInfo, output_path: pathlib.Path):
-    """Uses aiohttp to download an HTTP file.
-
-    Attempts to resume the file if `output_path` already exists.
-    """
-    logger.debug(f'Starting download of file {repr(str(url))}')
-
-    if output_path.is_file() and info.size == (output_path.stat().st_size * 8):
-        logger.warning(f'Already downloaded {repr(str(url))} to {repr(str(output_path))}')
-        return
-
-    if info.accept_ranges == 'bytes' or not output_path.is_file():
-        with open(output_path, 'ab') as fh:
-            headers = dict()
-            # Check the position of append, if it is 0 then we do not need to resume.
-            position = fh.tell()
-            if position:
-                headers['Range'] = f'bytes={position}-'
-
-            async with aiohttp.ClientSession() as session:
-                async with session.get(url, headers=headers) as response:
-                    if response.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:
-                        logger.warning(f'Server responded with 416, file is probably already downloaded')
-                        return
-
-                    if position and response.status != HTTPStatus.PARTIAL_CONTENT:
-                        raise UnrecoverableDownloadError(
-                            f'Tried to resume {repr(str(url))} but got status {response.status}')
-
-                    # May or may not be using Range.  Append each chunk to the output file.
-                    async for data in response.content.iter_any():
-                        fh.write(data)
-
-                        # TODO this cannot be canceled.
-                        # Sleep to catch cancel.
-                        await asyncio.sleep(0)
-    elif output_path.is_file():
-        # TODO support downloading files that cannot be resumed.
-        raise UnrecoverableDownloadError(f'Cannot resume download {url}')
-
-
 class FileDownloader(Downloader, ABC):
     """A Downloader which can download an arbitrary HTTP file.
 
@@ -135,7 +51,7 @@ class FileDownloader(Downloader, ABC):
         output_path = destination / info.name
 
         try:
-            await download_file(download.url, info, output_path)
+            await download_file(download.url, output_path, info)
             background_task(save_and_tag(output_path, download.settings.get('tag_names')))
 
             return DownloadResult(

wrolpi/files/lib.py

@@ -2,11 +2,13 @@ import asyncio
 import datetime
 import functools
 import glob
+import itertools
 import json
 import multiprocessing
 import os
 import pathlib
 import re
+import shutil
 import subprocess
 from itertools import zip_longest
 from pathlib import Path
@@ -21,14 +23,14 @@ from wrolpi import flags
 from wrolpi.cmd import which
 from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_concurrent, \
     partition, cancelable_wrapper, \
-    get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task
+    get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk
 from wrolpi.dates import now, from_timestamp
 from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
-from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag
+from wrolpi.errors import InvalidFile, UnknownDirectory, UnknownFile, UnknownTag, FileConflict, FileGroupIsTagged
 from wrolpi.events import Events
 from wrolpi.files.models import FileGroup, Directory
 from wrolpi.lang import ISO_639_CODES
-from wrolpi.tags import TagFile, Tag
+from wrolpi.tags import TagFile, Tag, tag_names_to_file_group_sub_select
 from wrolpi.vars import PYTEST
 
 try:
@@ -42,8 +44,9 @@ except ImportError:
 
 logger = logger.getChild(__name__)
 
-__all__ = ['list_directories_contents', 'delete_file', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
-           'get_mimetype', 'split_file_name_words', 'get_primary_file']
+__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
+           'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics', 'estimate_search',
+           'move', 'rename', 'remove_directory']
 
 
 @optional_session
@@ -131,12 +134,41 @@ def list_directories_contents(directories_: List[str]) -> Dict:
 
 
 @wrol_mode_check
-def delete_file(file: str):
-    """Delete a file in the media directory."""
-    file = get_media_directory() / file
-    if file.is_dir() or not file.is_file():
-        raise InvalidFile(f'Invalid file {file}')
-    file.unlink()
+def delete(*paths: Union[str, pathlib.Path]):
+    """
+    Delete a file or directory in the media directory.
+
+    This will refuse to delete any files (or files in the directories) that have been tagged."""
+    media_directory = get_media_directory()
+    paths = [media_directory / i for i in paths]
+    if any(i == media_directory for i in paths):
+        raise InvalidFile(f'Cannot delete the media directory')
+    for path in paths:
+        if not path.is_dir() and not path.is_file():
+            raise InvalidFile(f'Cannot delete {path} because it is not a file or a directory.')
+    if len(paths) > 1:
+        for p1, p2 in itertools.permutations(paths):
+            if str(p2).startswith(str(p1)):
+                raise InvalidFile(f'Cannot deleted nested paths')
+    with get_db_session() as session:
+        # Search for any files that have been tagged.
+        for path in paths:
+            query = session.query(FileGroup, TagFile) \
+                .join(TagFile, TagFile.file_group_id == FileGroup.id)
+            if path.is_file():
+                # Could be deleting a file in a FileGroup that has been tagged.
+                stem, _ = split_path_stem_and_suffix(path)
+                query = query.filter(FileGroup.primary_path.like(f'{path.parent}/{stem}%'))
+            else:
+                # Search for any FileGroups that have been tagged under this directory.
+                query = query.filter(FileGroup.primary_path.like(f'{path}/%'))
+            for (file_group, tag_file) in query:
+                raise FileGroupIsTagged(f"Cannot delete {file_group} because it is tagged")
+    for path in paths:
+        if path.is_dir():
+            remove_directory(path)
+        else:
+            path.unlink()
 
 
 FILE_NAME_REGEX = re.compile(r'[_ .]')
@@ -194,7 +226,6 @@ def get_mimetype(path: Path) -> str:
         output = subprocess.check_output(cmd)
         output = output.decode()
         mimetype = output.split(' ')[-1].strip()
-        return mimetype
     else:
         mimetype = mime.from_file(path)
     mimetype = _mimetype_suffix_map(path, mimetype)
@@ -432,6 +463,7 @@ async def refresh_discover_paths(paths: List[pathlib.Path], idempotency: datetim
         if deleted:
             # Delete any paths that do not exist.
             deleted_files = ' OR '.join([curs.mogrify('primary_path = %s', (str(i),)).decode() for i in deleted])
+
         idempotency = curs.mogrify('%s', (idempotency,)).decode()
         wheres = f' ( (idempotency != {idempotency} OR idempotency is null) AND ({wheres}))' if wheres else ''
         stmt = f'''
@@ -651,7 +683,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
 
     if tags:
         # Filter all FileGroups by those that have been tagged with the provided tag names.
-        tags_stmt, params_ = tag_names_to_sub_select(tags)
+        tags_stmt, params_ = tag_names_to_file_group_sub_select(tags)
         params.update(params_)
         wheres.append(f'fg.id = ANY({tags_stmt})')
 
@@ -683,24 +715,6 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
     return results, total
 
 
-def tag_names_to_sub_select(names: List[str]) -> Tuple[str, dict]:
-    """Create the SQL necessary to filter FileGroup by the provided Tag names."""
-    if not names:
-        return '', dict()
-
-    stmt = '''
-        SELECT
-            tf.file_group_id
-        FROM
-            tag_file tf
-            LEFT JOIN tag t on t.id = tf.tag_id
-        GROUP BY file_group_id
-        -- Match only FileGroups that have at least all the Tag names.
-        HAVING array_agg(t.name)::TEXT[] @> %(tag_names)s::TEXT[]
-    '''
-    return stmt, dict(tag_names=names)
-
-
 def handle_file_group_search_results(statement: str, params: dict) -> Tuple[List[dict], int]:
     """
     Execute the provided SQL statement and fetch the Files returned.
@@ -938,7 +952,8 @@ def get_refresh_progress():
     with get_db_curs() as curs:
         curs.execute('''
             SELECT
-                COUNT(id) AS "total_file_groups",
+                -- Sum all the files in each FileGroup.
+                SUM(json_array_length(files)) AS "total_file_groups",
                 COUNT(id) FILTER (WHERE indexed IS TRUE) AS "indexed",
                 COUNT(id) FILTER (WHERE indexed IS FALSE) AS "unindexed",
                 COUNT(id) FILTER (WHERE model IS NOT NULL) AS "modeled"
@@ -962,3 +977,196 @@ def get_refresh_progress():
         )
 
     return status
+
+
+async def estimate_search(search_str: str, tag_names: List[str]):
+    with get_db_curs() as curs:
+        if search_str and tag_names:
+            params = dict(search_str=search_str, tag_names=tag_names)
+            stmt = '''
+            SELECT COUNT(*) OVER() AS estimate
+            FROM file_group
+                LEFT JOIN tag_file tf on file_group.id = tf.file_group_id
+            WHERE textsearch @@ websearch_to_tsquery(%(search_str)s)
+                AND tf.tag_id IN (select id from tag where name = ANY(%(tag_names)s))
+            '''
+        elif tag_names:
+            params = dict(tag_names=tag_names)
+            stmt = '''
+            SELECT COUNT(*) OVER() AS estimate
+            FROM file_group
+                LEFT JOIN tag_file tf on file_group.id = tf.file_group_id
+            WHERE tf.tag_id IN (select id from tag where name = ANY(%(tag_names)s))'''
+        else:
+            params = dict(search_str=search_str)
+            stmt = '''
+            SELECT COUNT(*) OVER() AS estimate
+            FROM file_group
+            WHERE textsearch @@ websearch_to_tsquery(%(search_str)s)
+            '''
+        curs.execute(stmt, params)
+        result = curs.fetchone()
+        if result:
+            return int(result['estimate'])
+        return 0
+
+
+def _move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathlib.Path, pathlib.Path]]:
+    media_directory = get_media_directory()
+    for source in sources:
+        if not str(source).startswith(str(media_directory)):
+            raise FileNotFoundError(f'{source} is not within the media directory')
+    if not str(destination).startswith(str(media_directory)):
+        raise FileNotFoundError(f'{destination} is not within the media directory')
+
+    destination_existed = destination.is_dir()
+
+    # [ (pathlib.Path('source'), pathlib.Path('destination')), ... ]
+    plan = list()
+    unique_paths = set()
+    # Get all files that need to be moved.  Check for destination conflicts.
+    for source in sources:
+        if source.is_dir():
+            # Recursively move the directory's files.
+            for path in walk(source):
+                new_file = destination / path.relative_to(media_directory)
+                if new_file.exists():
+                    raise FileConflict(f'Cannot move {path} to {new_file} because it already exists')
+                if path not in unique_paths:
+                    plan.append((path, new_file))
+                    unique_paths.add(path)
+        elif source.is_file():
+            # Move all files associated with the source file.
+            for file in glob_shared_stem(source):
+                new_file = destination / file.name
+                if new_file.exists():
+                    raise FileConflict(f'Cannot move {file} to {new_file} because it already exists')
+                if file not in unique_paths:
+                    plan.append((file, new_file))
+                    unique_paths.add(file)
+
+    logger.info(f'Executing move plan with {len(plan)} steps.')
+
+    # Track the steps performed so they can be reversed.
+    revert_plan = list()
+
+    def do_plan(plan_):
+        # Move deepest files first.
+        plan_ = sorted(plan_, key=lambda i: (len(i[0].parents), i[0].name), reverse=True)
+        for old_file, new_file in plan_:  # noqa
+            new_file.parent.mkdir(parents=True, exist_ok=True)
+            logger.debug(f'Moving {old_file} to {new_file}')
+            # if not new_file.exists():
+            shutil.move(str(old_file), str(new_file))
+            logger.debug(f'Moved {old_file} to {new_file}')
+            revert_plan.append((new_file, old_file))
+        # Move is complete.
+        return plan_
+
+    try:
+        plan = do_plan(plan)
+        for source in sources:
+            if source.is_dir():
+                remove_directory(source)
+        logger.info(f'Move execution completed')
+        return plan
+    except Exception as e:
+        logger.error(f'Move failed', exc_info=e)
+        # Move files back.
+        do_plan(revert_plan)
+        # Delete the directories that were created.
+        directories = sorted(walk(destination), key=lambda i: len(i.parents), reverse=True)
+        for path in directories:
+            if path.is_dir():
+                remove_directory(path)
+        # Remove destination only if it did not exist at the start.
+        if destination.is_dir() and destination_existed is False:
+            remove_directory(destination)
+        raise
+
+
+def remove_directory(directory: pathlib.Path):
+    """Remove a directory (only if it is empty) remove it's Directory record."""
+    if directory.is_dir():
+        directory.rmdir()
+    with get_db_curs(commit=True) as curs:
+        stmt = 'DELETE FROM directory WHERE path=%s'
+        curs.execute(stmt, (str(directory),))
+
+
+async def move(destination: pathlib.Path, *sources: pathlib.Path) -> List[Tuple[pathlib.Path, pathlib.Path]]:
+    """Moves a file or directory (recursively), preserving applied Tags.
+
+    If the move fails at any point, the files will be returned to their previous locations.
+
+    Returns a list of tuples, the first Path in the tuple is the file's old location, the second Path is where the file
+    was moved.
+    """
+    # Move the files first, if this fails it will revert itself and raise an error.
+    plan = _move(destination, *sources)
+
+    # Files were moved, migrate any Tags.
+    plan_map = {old: new for old, new in plan}
+    with get_db_session(commit=True) as session:
+        old_file_paths = list(plan_map.keys())
+        old_files: List[Tuple[FileGroup, TagFile]] = session.query(FileGroup, TagFile) \
+            .filter(FileGroup.primary_path.in_(old_file_paths)) \
+            .join(TagFile, TagFile.file_group_id == FileGroup.id).all()
+        logger.info(f'Moving {len(old_files)} tagged files')
+        for old_file, _ in old_files:
+            try:
+                new_file_path = plan_map[old_file.primary_path]
+                logger.info(f'Moving tagged file {old_file.primary_path} to {new_file_path}')
+                old_file.move(new_file_path, move_files=False)
+            except KeyError as e:
+                # Got a duplicate?
+                logger.error('Error during tag migration', exc_info=e)
+        session.commit()
+
+    if PYTEST:
+        await refresh_files([destination, *sources])
+    else:
+        background_task(refresh_files([destination, *sources]))
+
+    return plan
+
+
+async def rename_file(path: pathlib.Path, new_name: str) -> pathlib.Path:
+    """Rename a file (and it's associated files).  Preserve any tags."""
+    new_path = path.with_name(new_name)
+    if not path.exists():
+        raise FileNotFoundError(f'Cannot find {path} to rename')
+    if new_path.exists():
+        raise FileConflict(f'Cannot rename {path} because {new_path} already exists')
+
+    with get_db_session(commit=True) as session:
+        fg = session.query(FileGroup).filter(FileGroup.primary_path == path).one_or_none()
+        if not fg:
+            # File wasn't yet in the DB.
+            fg = FileGroup.from_paths(session, path)
+        fg.move(new_path)
+
+    return new_path
+
+
+async def rename_directory(directory: pathlib.Path, new_name: str) -> pathlib.Path:
+    """Rename a directory.  This is done by moving all files into the new directory, and removing the old directory."""
+    new_directory = directory.with_name(new_name)
+    if new_directory.exists():
+        raise FileConflict(f'Cannot rename {directory} to {new_directory} because it already exists.')
+
+    # Move all paths into the new directory.
+    paths = list(directory.iterdir())
+    await move(new_directory, *paths)
+    # Remove the old directory.
+    remove_directory(directory)
+
+    return new_directory
+
+
+async def rename(path: pathlib.Path, new_name: str) -> pathlib.Path:
+    """Rename a directory or file.  Preserve any tags."""
+    if path.is_dir():
+        return await rename_directory(path, new_name)
+
+    return await rename_file(path, new_name)

wrolpi/files/models.py

@@ -1,4 +1,6 @@
+import copy
 import pathlib
+import shutil
 from datetime import datetime
 from typing import List, Type, Optional
 
@@ -289,6 +291,28 @@ class FileGroup(ModelHelper, Base):
 
         self.files = collected_files
 
+    def move(self, new_primary_path: pathlib.Path, move_files: bool = True):
+        """Move all files in this group to a new location."""
+        from wrolpi.files.lib import split_path_stem_and_suffix
+        if move_files and new_primary_path.exists():
+            raise FileExistsError(f'Cannot move {self} to {new_primary_path} because it already exists.')
+
+        new_name, _ = split_path_stem_and_suffix(new_primary_path, full=True)
+        # Need a deepcopy because changes to self.files are ignored otherwise.
+        new_files = copy.deepcopy(self.files)
+        for idx, file in enumerate(new_files):
+            _, suffix = split_path_stem_and_suffix(file['path'])
+            new_path = pathlib.Path(f'{new_name}{suffix}')
+            if move_files:
+                shutil.move(file['path'], new_path)
+            new_files[idx]['path'] = new_path
+        self.files = new_files
+        self.primary_path = new_primary_path
+        # Need to re-index for self.data.
+        self.indexed = False
+        # Flush the changes to the FileGroup.
+        Session.object_session(self).flush([self, ])
+
 
 class Directory(ModelHelper, Base):
     """A representation of a file directory in the media directory."""