rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-25 18:40:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Video downloads have many more options. Downloads are migrated.

Browse source 
Storing video settings in session storage to help user start downloads consistently.

Applying channel tag name to new channels when download videos or channels.

Initializing map tiles when importing and resetting map.
This commit is contained in:
Roland 2024-11-02 15:14:02 -06:00
parent a385d4f709
commit 5621d47309
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
77 changed files with 5878 additions and 2541 deletions
Download patch file Download diff file Expand all files Collapse all files

130
.circleci/config.yml
View file

@ -5,48 +5,6 @@ version: 2.1
# Define a job to be invoked later in a workflow.
# See: https://circleci.com/docs/2.0/configuration-reference/#jobs
jobs:
app-tests-18:
docker:
- image: cimg/node:18.12
resource_class: small
steps:
- checkout
- restore_cache:
key: dependency-cache-{{ checksum "app/package-lock.json" }}
- run: cd app && npm install --legacy-peer-deps
- save_cache:
key: dependency-cache-{{ checksum "app/package-lock.json" }}
paths:
- ./app/node_modules
- run: cd app && npm run test
app-tests-20:
docker:
- image: cimg/node:20.13
resource_class: small
steps:
- checkout
- restore_cache:
key: dependency-cache-{{ checksum "app/package-lock.json" }}
- run: cd app && npm install --legacy-peer-deps
- save_cache:
key: dependency-cache-{{ checksum "app/package-lock.json" }}
paths:
- ./app/node_modules
- run: cd app && npm run test
app-tests-22:
docker:
- image: cimg/node:22.1
resource_class: small
steps:
- checkout
- restore_cache:
key: dependency-cache-{{ checksum "app/package-lock.json" }}
- run: cd app && npm install --legacy-peer-deps
- save_cache:
key: dependency-cache-{{ checksum "app/package-lock.json" }}
paths:
- ./app/node_modules
- run: cd app && npm run test
api-tests-3-11:
docker:
- image: cimg/python:3.11
@ -156,32 +114,25 @@ jobs:
- run:
name: Stop Sanic API
command: pkill -f sanic
react-app-build:
react-app-start:
docker:
- image: cimg/node:20.13
resource_class: large
steps:
- checkout
- restore_cache:
key: dependency-cache-{{ checksum "app/package-lock.json" }}
key: app-{{ checksum "app/package-lock.json" }}
- run: cd app && npm install --legacy-peer-deps
- save_cache:
key: app-{{ checksum "app/package-lock.json" }}
paths:
- ./app/node_modules
- run:
name: Build React App
command: cd app && DISABLE_ESLINT_PLUGIN=true npm run build
react-app-start:
docker:
- image: cimg/node:20.13
resource_class: small
steps:
- checkout
- restore_cache:
key: dependency-cache-{{ checksum "app/package-lock.json" }}
- run: cd app && npm install --legacy-peer-deps
- run:
name: Start React App
command: |
cd app
npm run start
command: cd app && DISABLE_ESLINT_PLUGIN=true npm run start
background: true
- run:
name: Test React App
@ -191,6 +142,66 @@ jobs:
- run:
name: Stop React App
command: pkill -f npm
react-app-cypress-components-chrome:
docker:
- image: cypress/browsers:node-20.18.0-chrome-130.0.6723.69-1-ff-131.0.3-edge-130.0.2849.52-1
resource_class: small
steps:
- checkout
- restore_cache:
key: app-{{ checksum "app/package-lock.json" }}
- run: cd app && npm install --legacy-peer-deps
- run:
name: Run Cypress component tests
command: |
cd app
npx cypress run --component --browser chrome --reporter junit --reporter-options "mochaFile=test-results.xml"
- store_artifacts:
path: app/cypress/videos
- store_artifacts:
path: app/cypress/screenshots
- store_test_results:
path: app/test-results.xml
react-app-cypress-components-firefox:
docker:
- image: cypress/browsers:node-20.18.0-chrome-130.0.6723.69-1-ff-131.0.3-edge-130.0.2849.52-1
resource_class: small
steps:
- checkout
- restore_cache:
key: app-{{ checksum "app/package-lock.json" }}
- run: cd app && npm install --legacy-peer-deps
- run:
name: Run Cypress component tests
command: |
cd app
npx cypress run --component --browser firefox --reporter junit --reporter-options "mochaFile=test-results.xml"
- store_artifacts:
path: app/cypress/videos
- store_artifacts:
path: app/cypress/screenshots
- store_test_results:
path: app/test-results.xml
react-app-cypress-components-edge:
docker:
- image: cypress/browsers:node-20.18.0-chrome-130.0.6723.69-1-ff-131.0.3-edge-130.0.2849.52-1
resource_class: small
steps:
- checkout
- restore_cache:
key: app-{{ checksum "app/package-lock.json" }}
- run: cd app && npm install --legacy-peer-deps
- run:
name: Run Cypress component tests
command: |
cd app
npx cypress run --component --browser edge --reporter junit --reporter-options "mochaFile=test-results.xml"
- store_artifacts:
path: app/cypress/videos
- store_artifacts:
path: app/cypress/screenshots
- store_test_results:
path: app/test-results.xml
workflows:
wrolpi-api-tests:
@ -201,8 +212,7 @@ workflows:
# - api-tests-3-12 Sanic does not yet support 3.12.
wrolpi-app-test:
jobs:
- app-tests-18
- app-tests-20
- app-tests-22
- react-app-build
- react-app-start
- react-app-cypress-components-chrome
- react-app-cypress-components-firefox
- react-app-cypress-components-edge

10
.run/All Tests.run.xml
View file

@ -1,10 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="All Tests" type="JavaScriptTestRunnerJest" nameIsGenerated="true">
<node-interpreter value="project" />
<jest-package value="$PROJECT_DIR$/app/node_modules/react-scripts" />
<working-dir value="$PROJECT_DIR$/app" />
<envs />
<scope-kind value="ALL" />
<method v="2" />
</configuration>
</component>

35
alembic/versions/170d1be52bc7_download_destination.py Normal file
View file

@ -0,0 +1,35 @@
"""Create Download.destination column.
Revision ID: 170d1be52bc7
Revises: 9ec4c765ef8d
Create Date: 2024-11-02 11:16:15.297850
"""
import os
from alembic import op
from sqlalchemy.orm import Session
# revision identifiers, used by Alembic.
revision = '170d1be52bc7'
down_revision = '9ec4c765ef8d'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS destination TEXT')
session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS tag_names TEXT[]')
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
session.execute('ALTER TABLE download DROP COLUMN IF EXISTS tag_names')
session.execute('ALTER TABLE download DROP COLUMN IF EXISTS destination')

137
alembic/versions/1f2325523525_download_destination_migrate.py Normal file
View file

@ -0,0 +1,137 @@
"""Create `Download.destination` and `Download.tag_names`, migrate from `Download.settings`.
Revision ID: 1f2325523525
Revises: 170d1be52bc7
Create Date: 2024-11-02 11:20:45.028865
"""
import os
from copy import copy
from alembic import op
from sqlalchemy import Column, Integer, Text, String, ARRAY
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from wrolpi.common import ModelHelper
from wrolpi.dates import TZDateTime
from wrolpi.downloader import get_download_manager_config, DownloadStatus
# revision identifiers, used by Alembic.
revision = '1f2325523525'
down_revision = '170d1be52bc7'
branch_labels = None
depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
Base = declarative_base()
class MDownload(ModelHelper, Base):
__tablename__ = 'download'
id = Column(Integer, primary_key=True)
url = Column(String, nullable=False, unique=True)
attempts = Column(Integer, default=0)
destination = Column(Text)
downloader = Column(Text)
sub_downloader = Column(Text)
error = Column(Text)
frequency = Column(Integer)
info_json = Column(JSONB)
last_successful_download = Column(TZDateTime)
location = Column(Text)
next_download = Column(TZDateTime)
settings = Column(JSONB)
status = Column(String, default=DownloadStatus.new)
tag_names = Column(ARRAY(Text))
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
for download in session.query(MDownload):
if not download.settings:
continue
# Move `download.settings['destination']` to `download.destination`, if any.
settings = copy(download.settings) if download.settings else dict()
destination = settings.pop('destination', None)
tag_names = settings.pop('tag_names', None)
if isinstance(excluded_urls := settings.get('excluded_urls'), list):
settings['excluded_urls'] = ','.join(excluded_urls)
download.destination = destination or None
download.tag_names = tag_names or None
download.settings = settings
# This will migrate the config as-is at the time that this migration was written. This code is probably not
# safe to reuse in the future!
# Read config file directly.
try:
config = get_download_manager_config()
config._config.update(config.read_config_file())
except FileNotFoundError:
# Config does not exist, probably testing, or a new WROLPi.
return
new_downloads = []
for download in session.query(MDownload).order_by(MDownload.url):
if download.last_successful_download and not download.frequency:
# This once-download has completed, do not save it.
continue
new_download = dict(
destination=download.destination,
downloader=download.downloader,
frequency=download.frequency,
last_successful_download=download.last_successful_download,
next_download=download.next_download,
settings=download.settings,
status=download.status,
sub_downloader=download.sub_downloader,
url=download.url,
)
new_downloads.append(new_download)
# Write directly to file, bypassing usual checks.
config._config['downloads'] = new_downloads
config.write_config_data(config._config, config.get_file())
def downgrade():
bind = op.get_bind()
session = Session(bind=bind)
for download in session.query(MDownload):
if download.destination:
settings = copy(download.settings) if download.settings else dict()
settings['destination'] = download.destination or settings.get('destination')
settings['tag_names'] = download.tag_names or settings.get('tag_names')
if isinstance(excluded_urls := settings.get('excluded_urls'), str):
settings['excluded_urls'] = excluded_urls.split(',')
download.settings = settings
try:
config = get_download_manager_config()
config._config.update(config.read_config_file())
except FileNotFoundError:
# Config does not exist, probably testing, or a new WROLPi.
return
new_downloads = []
for download in session.query(MDownload).order_by(MDownload.url):
new_download = dict(
downloader=download.downloader,
frequency=download.frequency,
last_successful_download=download.last_successful_download,
next_download=download.next_download,
settings=download.settings,
status=download.status,
sub_downloader=download.sub_downloader,
url=download.url,
)
new_downloads.append(new_download)
config._config['downloads'] = new_downloads
config.write_config_data(config._config, config.get_file())

34
alembic/versions/8d0d81bc9c34_channel_channel_downloads.py
View file

@ -9,12 +9,16 @@ import os
import pathlib
from alembic import op
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date
from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, Text, ForeignKey
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session, deferred
from sqlalchemy.orm import Session, deferred, relationship
from sqlalchemy.orm.collections import InstrumentedList
from modules.videos.lib import link_channel_and_downloads
from wrolpi.common import ModelHelper
from wrolpi.dates import TZDateTime
from wrolpi.downloader import DownloadStatus
from wrolpi.media_path import MediaPathType
# revision identifiers, used by Alembic.
@ -43,6 +47,8 @@ class MChannel(ModelHelper, Base):
info_json = deferred(Column(JSON))
info_date = Column(Date)
downloads: InstrumentedList = relationship('MDownload', primaryjoin='MDownload.channel_id==MChannel.id')
@staticmethod
def get_by_url(url: str, session: Session = None):
if not url:
@ -55,6 +61,28 @@ class MChannel(ModelHelper, Base):
return f'https://www.youtube.com/feeds/videos.xml?channel_id={self.source_id}'
# The `Download` model at the time of this migration.
class MDownload(ModelHelper, Base):
__tablename__ = 'download'
id = Column(Integer, primary_key=True)
url = Column(String, nullable=False, unique=True)
attempts = Column(Integer, default=0)
downloader = Column(Text)
sub_downloader = Column(Text)
error = Column(Text)
frequency = Column(Integer)
info_json = Column(JSONB)
last_successful_download = Column(TZDateTime)
location = Column(Text)
next_download = Column(TZDateTime)
settings = Column(JSONB)
status = Column(String, default=DownloadStatus.new)
channel_id = Column(Integer, ForeignKey('channel.id'))
channel = relationship('MChannel', primaryjoin='MDownload.channel_id==MChannel.id', back_populates='downloads')
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
@ -62,7 +90,7 @@ def upgrade():
session.execute('ALTER TABLE download ADD CONSTRAINT download_url_unique UNIQUE (url)')
session.execute('ALTER TABLE download ADD COLUMN channel_id INTEGER REFERENCES channel(id)')
link_channel_and_downloads(session, MChannel)
link_channel_and_downloads(session, MChannel, MDownload)
session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS match_regex')
session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS download_frequency')

10
app/cypress.config.js Normal file
View file

@ -0,0 +1,10 @@
const { defineConfig } = require("cypress");
module.exports = defineConfig({
component: {
devServer: {
framework: "create-react-app",
bundler: "webpack",
},
},
});

5
app/cypress/fixtures/example.json Normal file
View file

@ -0,0 +1,5 @@
{
"name": "Using fixtures to represent data",
"email": "hello@cypress.io",
"body": "Fixtures are a great way to mock data for responses to routes"
}

18
app/cypress/fixtures/tagsMock.json Normal file
View file

@ -0,0 +1,18 @@
{
"tags": [
{
"id": 1,
"name": "Automotive",
"color": "#353535",
"file_group_count": 0,
"zim_entry_count": 0
},
{
"id": 2,
"name": "Clothing",
"color": "#9858C5",
"file_group_count": 0,
"zim_entry_count": 0
}
]
}

4
app/cypress/support/commands.js Normal file
View file

@ -0,0 +1,4 @@
import {mount} from 'cypress/react18'
// Use React 18 for all mounts.
Cypress.Commands.add('mount', mount);

12
app/cypress/support/component-index.html Normal file
View file

@ -0,0 +1,12 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width,initial-scale=1.0">
<title>Components App</title>
</head>
<body>
<div data-cy-root></div>
</body>
</html>

54
app/cypress/support/component.js Normal file
View file

@ -0,0 +1,54 @@
// ***********************************************************
// This example support/component.js is processed and
// loaded automatically before your test files.
//
// This is a great place to put global configuration and
// behavior that modifies Cypress.
//
// You can change the location of this file or turn off
// automatically serving support files with the
// 'supportFile' configuration option.
//
// You can read more here:
// https://on.cypress.io/configuration
// ***********************************************************
import './commands'
import 'semantic-ui-offline/semantic.min.css';
import {MemoryRouter, Route, Routes} from "react-router-dom";
import {QueryProvider} from "../../src/hooks/customHooks";
import {TagsProvider} from "../../src/Tags";
import React from "react";
Cypress.Commands.add('mountWithRouter', (component, options) => {
options = options || {};
const initialEntries = options?.initialEntries || ['/'];
return cy.mount(
<MemoryRouter initialEntries={initialEntries}>
<QueryProvider>
<Routes>
<Route path='/videos/channels/new' element={component}/>
<Route path='/videos/channels/:channelId' element={component}/>
<Route path='*' element={component}/>
</Routes>
</QueryProvider>
</MemoryRouter>,
options
);
});
Cypress.Commands.add('mountWithTags', (component, options) => {
cy.fixture('tagsMock.json').then((mockTags) => {
cy.intercept('GET', '**/api/tag', {
statusCode: 200,
body: mockTags
}).as('getTags');
});
return cy.mountWithRouter(
<TagsProvider>
{component}
</TagsProvider>,
options
);
});

2778
app/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

6
app/package.json
View file

@ -4,9 +4,10 @@
"private": true,
"dependencies": {
"@artsy/fresnel": "^6.1.0",
"@testing-library/jest-dom": "^5.16.5",
"@cypress/react18": "^2.0.1",
"@testing-library/react": "^13.4.0",
"@testing-library/user-event": "^13.5.0",
"cypress": "^13.15.2",
"lodash": "^4.17.21",
"query-string": "^8.1.0",
"react": "^18.2.0",
@ -31,8 +32,7 @@
},
"eslintConfig": {
"extends": [
"react-app",
"react-app/jest"
"react-app"
]
},
"browserslist": {

3
app/src/App.js
View file

@ -21,7 +21,8 @@ import {SemanticToastContainer} from "react-semantic-toasts-2";
import {FilePreviewProvider} from "./components/FilePreview";
import {TagsProvider} from "./Tags";
import {ZimRoute} from "./components/Zim";
import {HELP_VIEWER_URI, IframeViewer} from "./components/Common";
import {IframeViewer} from "./components/Common";
import {HELP_VIEWER_URI} from "./components/Vars";
function PageNotFound() {
const {t} = useContext(ThemeContext);

9
app/src/App.test.js
View file

@ -1,9 +0,0 @@
import React from "react";
import {render} from '@testing-library/react';
import '@testing-library/jest-dom';
import App from "./App";
test('renders without crashing', () => {
render(<App/>);
});

2
app/src/DashboardPage.js
View file

@ -32,7 +32,7 @@ function FlagsMessages() {
const {settings, fetchSettings} = React.useContext(SettingsContext);
const {status} = useContext(StatusContext);
if (!status || !'flags' in status || _.isEmpty(status['flags'])) {
if (_.isEmpty(status?.flags)) {
return <></>
}

1
app/src/Events.js
View file

@ -2,7 +2,6 @@ import React, {useEffect} from 'react';
import {useRecurringTimeout} from "./hooks/customHooks";
import {getEvents} from "./api";
import {toast} from "react-semantic-toasts-2";
import {useLocalStorage} from "./components/Common";
const apiEventName = 'apiEvent';

15
app/src/Tags.cy.js Normal file
View file

@ -0,0 +1,15 @@
import React from "react";
import {TagsDashboard} from "./Tags";
// Mock data for tags
describe('Tags', () => {
beforeEach(() => {
cy.mountWithTags(<TagsDashboard/>, {});
});
it('Tags Dashboard displays Tags.', () => {
cy.wait('@getTags');
cy.get('div.ui.label.large').contains('Automotive').should('be.visible');
});
});

20
app/src/Tags.js
View file

@ -1,6 +1,6 @@
import React, {useEffect} from "react";
import {deleteTag, getTags, saveTag} from "./api";
import {Dimmer, Divider, Form, Grid, GridColumn, GridRow, Input, Label, TableCell, TableRow,} from "semantic-ui-react";
import {Dimmer, Divider, Form, Grid, GridColumn, GridRow, Label, TableCell, TableRow,} from "semantic-ui-react";
import {APIButton, contrastingColor, ErrorMessage, fuzzyMatch, scrollToTopOfElement} from "./components/Common";
import {
Button,
@ -75,7 +75,6 @@ export function useTags() {
}
const NameToTagLabel = ({name, to, ...props}) => {
const defaultTag = <Label size='large'>{name}</Label>;
const tag = findTagByName(name);
if (tag !== null && tag !== undefined) {
const tagColor = tag['color'] || DEFAULT_TAG_COLOR;
@ -92,7 +91,7 @@ export function useTags() {
}
// No tags have been fetched.
return defaultTag;
return <Label size='large'>{name}</Label>;
}
const TagsGroup = ({tagNames, onClick}) => {
@ -326,7 +325,9 @@ export function AddTagsButton({
onAdd = _.noop,
onRemove = _.noop,
onChange = _.noop, // Expects to send: (tagNames, anyTag)
closeAfterLimit = true,
limit = null,
disabled = false,
}) {
// A button which displays a modal in which the user can add or remove tags.
@ -347,13 +348,16 @@ export function AddTagsButton({
const addTag = (name) => {
setLoading(true);
try {
const newTags = [...localTags, name];
const newTags = [...(localTags || []), name];
if (limit !== null && newTags.length > limit) {
return;
}
setLocalTags(newTags);
onAdd(name);
onChange(newTags, null);
if (closeAfterLimit && newTags && limit && newTags.length >= limit) {
setOpen(false);
}
} finally {
setLoading(false);
}
@ -399,6 +403,8 @@ export function AddTagsButton({
icon={active ? 'tags' : 'tag'}
color={active ? 'violet' : undefined}
onClick={handleOpen}
type="button"
disabled={disabled}
/>
<Modal closeIcon
open={open}
@ -443,7 +449,9 @@ export const TagsSelector = ({
onAdd = _.noop,
onRemove = _.noop,
onChange = _.noop,
closeAfterLimit = true,
limit = null,
disabled = false,
}) => {
// Provides a button to add tags to a list. Displays the tags of that list.
const {TagsLinkGroup} = React.useContext(TagsContext);
@ -460,15 +468,17 @@ export const TagsSelector = ({
onAdd={onAdd}
onRemove={onRemove}
onChange={onChange}
closeAfterLimit={closeAfterLimit}
anyTag={anyTag}
limit={limit}
disabled={disabled}
/>;
if (hideGroup) {
return button;
}
return <Grid>
return <Grid columns={2}>
<Grid.Row>
<Grid.Column mobile={2} computer={1}>
{button}

96
app/src/api.js
View file

@ -1,6 +1,7 @@
import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, emptyToNull, OTP_API, VIDEOS_API, ZIM_API} from "./components/Common";
import {emptyToNull} from "./components/Common";
import {toast} from "react-semantic-toasts-2";
import _ from "lodash";
import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, Downloaders, OTP_API, VIDEOS_API, ZIM_API} from "./components/Vars";
function timeoutPromise(ms, promise) {
// Create a timeout wrapper around a promise. If the timeout is reached, throw an error. Otherwise, return
@ -229,19 +230,8 @@ export async function deleteVideos(videoIds) {
}
export async function downloadVideoMetadata(videoUrl, destination) {
return await postDownload(
[videoUrl],
'video',
null,
null,
null,
destination,
null,
null,
null,
null,
true,
);
const downloadData = {urls: [videoUrl], downloader: Downloaders.Video, destination: destination};
return await postDownload(downloadData);
}
export async function getDirectories(search_str) {
@ -385,6 +375,19 @@ export async function postSaveConfig(fileName) {
return response
}
export async function fetchVideoDownloaderConfig() {
const response = await apiGet(`${API_URI}/config/videos_downloader.yaml`);
return (await response.json())['config'];
}
export async function saveVideoDownloaderConfig(config) {
const body = {config};
const response = await apiPost(`${API_URI}/config/videos_downloader.yaml`, body);
if (!response.ok) {
throw Error('Failed to save config');
}
}
export async function validateRegex(regex) {
const response = await apiPost(`${API_URI}/valid_regex`, {regex});
try {
@ -750,32 +753,8 @@ export async function getArchive(archiveId) {
}
}
function calculateDownloadSettings(excludedURLs, depth, suffix, tagNames, downloadMetadataOnly, destination, max_pages) {
let settings = {};
if (excludedURLs) settings['excluded_urls'] = excludedURLs;
if (depth) settings['depth'] = depth;
if (suffix) settings['suffix'] = suffix;
if (tagNames) settings['tag_names'] = tagNames;
if (downloadMetadataOnly) settings['download_metadata_only'] = downloadMetadataOnly;
if (destination) settings['destination'] = destination;
if (max_pages) settings['max_pages'] = max_pages;
return settings;
}
export async function postDownload(
urls,
downloader,
frequency,
sub_downloader,
excludedURLs,
destination,
tagNames,
depth,
suffix,
max_pages,
downloadMetadataOnly,
) {
if (!downloader) {
export async function postDownload(downloadData) {
if (!downloadData.downloader) {
toast({
type: 'error',
title: 'Failed to submit download',
@ -785,14 +764,14 @@ export async function postDownload(
throw new Error('downloader is required, but was not provided');
}
let settings = calculateDownloadSettings(excludedURLs, depth, suffix, tagNames, downloadMetadataOnly, destination,
max_pages);
let body = {
urls: urls,
downloader: downloader,
sub_downloader: sub_downloader,
frequency: frequency || null,
settings: settings,
urls: downloadData.urls,
destination: downloadData.destination,
downloader: downloadData.downloader,
sub_downloader: downloadData.sub_downloader || null,
frequency: downloadData.frequency || null,
tag_names: downloadData.tag_names || null,
settings: downloadData.settings,
};
const response = await apiPost(`${API_URI}/download`, body);
if (!response.ok) {
@ -800,19 +779,13 @@ export async function postDownload(
toast({
type: 'error', title: 'Download Error', description: message, time: 5000,
});
throw new Error('Failed to create download');
}
return response;
}
export async function putDownload(
urls,
download_id,
downloader,
sub_downloader,
frequency,
excludedURLs,
) {
if (!downloader) {
export async function putDownload(download_id, downloadData) {
if (!downloadData.downloader) {
toast({
type: 'error',
title: 'Failed to submit download',
@ -822,20 +795,13 @@ export async function putDownload(
throw new Error('downloader is required, but was not provided');
}
let settings = calculateDownloadSettings(excludedURLs)
let body = {
urls: urls,
downloader: downloader,
sub_downloader: sub_downloader,
frequency: frequency || null,
settings: settings,
};
const response = await apiPut(`${API_URI}/download/${download_id}`, body);
const response = await apiPut(`${API_URI}/download/${download_id}`, downloadData);
if (!response.ok) {
const message = await getErrorMessage(response, 'Unable to update Download. See server logs.');
toast({
type: 'error', title: 'Download Error', description: message, time: 5000,
});
throw new Error('Failed to update download');
}
return response;
}

19
app/src/components/Archive.js
View file

@ -27,7 +27,6 @@ import {
HelpHeader,
humanFileSize,
isoDatetimeToAgoPopup,
isoDatetimeToString,
mimetypeColor,
PageContainer,
PreviewPath,
@ -49,6 +48,7 @@ import {Button, Card, CardIcon, darkTheme, Header, Loader, Placeholder, Popup, S
import {SortableTable} from "./SortableTable";
import {taggedImageLabel, TagsSelector} from "../Tags";
import {toast} from "react-semantic-toasts-2";
import {Downloaders} from "./Vars";
function archiveFileLink(path, directory = false) {
if (path) {
@ -111,7 +111,8 @@ function ArchivePage() {
}
const localUpdateArchive = async () => {
const response = await postDownload([archiveFile.url], 'archive');
const downloadData = {urls: [archiveFile.url], downloader: Downloaders.Archive};
const response = await postDownload(downloadData);
if (response.ok) {
toast({
type: 'success',
@ -170,11 +171,11 @@ function ArchivePage() {
await fetchArchive();
}
const archivedDatetimeString = isoDatetimeToString(archiveFile.download_datetime, true);
const publishedDatetimeString = archiveFile.published_datetime ? isoDatetimeToString(archiveFile.published_datetime, true)
const archivedDatetimeString = isoDatetimeToAgoPopup(archiveFile.download_datetime, true);
const publishedDatetimeString = archiveFile.published_datetime ? isoDatetimeToAgoPopup(archiveFile.published_datetime, true)
: 'unknown';
const modifiedDatetimeString = archiveFile.published_modified_datetime
? isoDatetimeToString(archiveFile.published_modified_datetime, true)
? isoDatetimeToAgoPopup(archiveFile.published_modified_datetime, true)
: 'unknown';
const aboutPane = {
@ -568,15 +569,15 @@ export function ArchiveRowCells({file}) {
poster = <FileIcon file={file} size='large'/>;
}
let dataCell = file.published_datetime ? isoDatetimeToString(file.published_datetime) : '';
let dataCell = file.published_datetime ? isoDatetimeToAgoPopup(file.published_datetime) : '';
if (sort === 'published_modified_datetime') {
dataCell = file.published_modified_datetime ? isoDatetimeToString(file.published_modified_datetime) : '';
dataCell = file.published_modified_datetime ? isoDatetimeToAgoPopup(file.published_modified_datetime) : '';
} else if (sort === 'download_datetime') {
dataCell = file.download_datetime ? isoDatetimeToString(file.download_datetime) : '';
dataCell = file.download_datetime ? isoDatetimeToAgoPopup(file.download_datetime) : '';
} else if (sort === 'size') {
dataCell = humanFileSize(file.size);
} else if (sort === 'viewed') {
dataCell = isoDatetimeToString(file.viewed);
dataCell = isoDatetimeToAgoPopup(file.viewed);
}
// Fragment for SelectableRow

2
app/src/components/Calculators.js
View file

@ -1,6 +1,6 @@
import React from "react";
import {Breadcrumbs, useTitle} from "./Common";
import {Button, Icon, Segment} from "./Theme";
import {Button, Icon} from "./Theme";
import {useCalcQuery} from "../hooks/customHooks";
import {TemperatureCalculator} from "./calculators/TemperatureCalculator";
import {ElectricalCalculators} from "./calculators/ElectricalCalculators";

58
app/src/components/Channels.cy.js Normal file
View file

@ -0,0 +1,58 @@
import React from "react";
import {ChannelPage} from "./Channels";
describe(' Channels Page', () => {
beforeEach(() => {
// Do not return directories for these tests.
cy.intercept('POST', '**/api/files/search_directories', {
statusCode: 200,
body: {
is_dir: true, channel_directories: [], domain_directories: [],
directories: [{path: "some directory", name: "some directory"}],
},
}).as('searchDirectories');
});
it('New Channel page can be rendered', () => {
cy.mountWithTags(
<ChannelPage create={true}/>,
{initialEntries: ['/videos/channels/new']}
);
cy.get('input[name="name"]').should('be.visible');
cy.wait('@searchDirectories');
});
it('Edit Channel page can be rendered', () => {
cy.intercept('GET', '**/api/videos/channels/1', {
statusCode: 200,
body: {
channel: {
name: 'Editing Channel',
tag_name: 'Automotive',
id: 1,
directory: 'some directory',
rss_url: 'https://example.com/rss-feed',
downloads: [],
statistics: {
size: 1,
largest_video: 2,
video_count: 3,
length: 4,
video_tags: 5,
}
}
},
}).as('channelFetch');
cy.mountWithTags(
<ChannelPage create={false}/>,
{initialEntries: ['/videos/channels/1']}
);
cy.wait('@channelFetch');
cy.wait('@searchDirectories'); // Needed for CircleCi.
cy.wait(500);
cy.get('input[name="name"]').should('be.visible');
cy.get('input[name="name"]').should('have.value', 'Editing Channel');
cy.get('div.ui.label.large').contains('Automotive').should('be.visible');
});
});

389
app/src/components/Channels.js
View file

@ -1,44 +1,27 @@
import React, {useState} from "react";
import {Dimmer, Dropdown, Grid, Input, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react";
import {
createChannel,
createChannelDownload,
deleteChannel,
deleteDownload,
refreshChannel,
tagChannel,
tagChannelInfo,
updateChannel,
updateChannelDownload
} from "../api";
import {Grid, Input, StatisticLabel, StatisticValue, TableCell, TableRow,} from "semantic-ui-react";
import {createChannel, deleteChannel, refreshChannel, tagChannel, tagChannelInfo, updateChannel} from "../api";
import {
APIButton,
BackButton,
DirectorySearch,
ErrorMessage,
frequencyOptions,
HelpHeader,
humanFileSize,
humanNumber,
RequiredAsterisk,
SearchInput,
secondsToFrequency,
secondsToFullDuration,
SimpleAccordion,
Toggle,
useTitle,
validURL,
WROLModeMessage
} from "./Common";
import {Link, useNavigate, useParams} from "react-router-dom";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useChannel, useChannels, useOneQuery, useWROLMode} from "../hooks/customHooks";
import {useChannel, useChannels, useOneQuery} from "../hooks/customHooks";
import _ from "lodash";
import {
Button,
Form,
FormField,
FormGroup,
FormInput,
Header,
Loader,
Modal,
@ -53,6 +36,8 @@ import {SortableTable} from "./SortableTable";
import {toast} from "react-semantic-toasts-2";
import {RecurringDownloadsTable} from "./admin/Downloads";
import {TagsContext, TagsSelector} from "../Tags";
import {InputForm} from "../hooks/useForm";
import {ChannelDownloadForm, DestinationForm, DownloadTagsSelector} from "./Download";
function ChannelStatistics({statistics}) {
@ -86,17 +71,17 @@ function ChannelStatistics({statistics}) {
}
function ChannelPage({create, header}) {
const [disabled, setDisabled] = useState(false);
export function ChannelPage({create, header}) {
// Used to display messages to maintainer.
const [error, setError] = useState(false);
const [success, setSuccess] = useState(false);
const [messageHeader, setMessageHeader] = useState();
const [messageContent, setMessageContent] = useState();
const [downloadModalOpen, setDownloadModalOpen] = useState(false);
const navigate = useNavigate();
const {channelId} = useParams();
const {channel, ready: channelReady, changeValue, fetchChannel} = useChannel(channelId);
const {SingleTag} = React.useContext(TagsContext);
const [tagEditModalOpen, setTagEditModalOpen] = useState(false);
@ -104,24 +89,21 @@ function ChannelPage({create, header}) {
const [moveToTagDirectory, setMoveToTagDirectory] = useState(true);
const [newTagDirectory, setNewTagDirectory] = useState('');
const {channel, form, fetchChannel} = useChannel(channelId);
useTitle(_.isEmpty(channel) ? null : `${channel.name} Channel`);
React.useEffect(() => {
setNewTagName(channel.tag_name);
}, [channel.tag_name]);
if (channel && channel.tag_name !== newTagName) {
setNewTagName(channel.tag_name);
}
}, [channel]);
if (!create && !channelReady) {
if (!create && !form.ready) {
// Waiting for editing Channel to be fetched.
return <Loader active/>;
}
const handleInputChange = (e, {name, value}) => {
if (e) {
e.preventDefault();
}
changeValue(name, value);
}
const setErrorMessage = (header, message) => {
setError(true);
setSuccess(false);
@ -136,18 +118,13 @@ function ChannelPage({create, header}) {
setMessageContent(message);
}
const handleSubmit = async (e) => {
if (e) {
e.preventDefault();
}
const handleSubmit = async () => {
const body = {
name: channel.name,
directory: channel.directory,
url: channel.url,
};
setDisabled(true);
let response = null;
try {
if (create !== undefined) {
@ -166,8 +143,6 @@ function ChannelPage({create, header}) {
time: 5000,
});
return;
} finally {
setDisabled(false);
}
if (response && response.ok) {
@ -237,7 +212,12 @@ function ChannelPage({create, header}) {
let downloads = channel && channel.downloads ? channel.downloads : null;
const afterNewDownloadSave = async () => {
await fetchChannel();
setDownloadModalOpen(false);
await form.fetcher();
}
const onDelete = async () => {
await form.fetcher();
setDownloadModalOpen(false);
}
@ -318,6 +298,54 @@ function ChannelPage({create, header}) {
</Modal>;
}
let channelUrlRow;
if (channel && channel.url) {
channelUrlRow = <Grid.Row columns={1}>
<Grid.Column>
<Header as='h4'>URL</Header>
<a href={channel.url}>{channel.url}</a>
</Grid.Column>
</Grid.Row>;
}
let channelRssUrlRow;
if (channel && channel.rss_url) {
channelRssUrlRow = <Grid.Row columns={1}>
<Grid.Column>
<Header as='h4'>RSS URL</Header>
<a href={channel.rss_url}>{channel.rss_url}</a>
</Grid.Column>
</Grid.Row>;
}
let channelTagRow;
if (channel && channel.tag_name) {
channelTagRow = <Grid.Row columns={1}>
<Grid.Column>
{channel.tag_name && !create && <SingleTag name={channel.tag_name}/>}
</Grid.Column>
</Grid.Row>;
}
let messageRow;
if (error || success) {
messageRow = <Grid.Row columns={1}>
<Grid.Column>
{error &&
<Message negative
header={messageHeader}
content={messageContent}
/>}
{success &&
<Message positive
header={messageHeader}
content={messageContent}
/>}
</Grid.Column>
</Grid.Row>
}
return <>
<BackButton/>
{!create &&
@ -334,56 +362,37 @@ function ChannelPage({create, header}) {
success={success}
autoComplete="off"
>
<FormGroup>
<FormField width={8}>
<FormInput required
label="Channel Name"
name="name"
type="text"
placeholder="Short Channel Name"
disabled={disabled}
value={channel.name}
onChange={(e, {value}) => changeValue('name', value)}
/>
</FormField>
<FormField width={8}>
<label>
Directory <RequiredAsterisk/>
</label>
<DirectorySearch
value={channel.directory}
onSelect={value => changeValue('directory', value)}
placeholder='videos/channel directory'
/>
</FormField>
</FormGroup>
<FormGroup>
<FormField width={16}>
<FormInput
label="URL"
name="url"
type="url"
placeholder='https://example.com/channel/videos'
value={channel.url}
onChange={handleInputChange}
/>
</FormField>
</FormGroup>
{channel.tag_name && !create && <SingleTag name={channel.tag_name}/>}
<Message error
header={messageHeader}
content={messageContent}
/>
<Message success
header={messageHeader}
content={messageContent}
/>
<Grid stackable columns={2}>
<Grid.Row style={{marginTop: '1em'}}>
<Grid.Row>
<Grid.Column>
<InputForm
form={form}
label="Channel Name"
name="name"
placeholder="Short Channel Name"
required={true}
/>
</Grid.Column>
<Grid.Column>
<DestinationForm
form={form}
label='Directory'
name='directory'
path='directory'
required={true}
/>
</Grid.Column>
</Grid.Row>
{channelTagRow}
{!create && (channel.url || channel.rss_url) &&
<SimpleAccordion title='Details'>
<Grid>
{channelUrlRow}
{channelRssUrlRow}
</Grid>
</SimpleAccordion>}
{messageRow}
<Grid.Row>
<Grid.Column width={8}>
{!create ?
<>
@ -411,14 +420,12 @@ function ChannelPage({create, header}) {
>Tag</Button>
{tagEditModal}
</>
: <>
<TagsSelector
limit={1}
selectedTagNames={channel.tag_name ? [channel.tag_name] : []}
onAdd={i => changeValue('tag_name', i)}
onRemove={() => changeValue('tag_name', null)}
/>
</>
: <DownloadTagsSelector
form={form}
limit={1}
path='tag_name'
name='tag_name'
/>
}
</Grid.Column>
<Grid.Column>
@ -427,7 +434,7 @@ function ChannelPage({create, header}) {
size='big'
floated='right'
onClick={handleSubmit}
disabled={disabled}
disabled={form.disabled}
obeyWROLMode={true}
>Save</APIButton>
</Grid.Column>
@ -457,190 +464,20 @@ function ChannelPage({create, header}) {
<Header as='h2'>New Channel Download</Header>
<ChannelDownloadForm
channel_id={channelId}
afterSave={afterNewDownloadSave}
closeModal={() => setDownloadModalOpen(false)}
onSuccess={afterNewDownloadSave}
onCancel={() => setDownloadModalOpen(false)}
onDelete={onDelete}
/>
</ModalContent>
</Modal>
<RecurringDownloadsTable downloads={downloads} fetchDownloads={fetchChannel}/>
<RecurringDownloadsTable downloads={downloads} fetchDownloads={fetchChannel} onDelete={onDelete}/>
</Segment>}
<div style={{marginTop: '2em'}}>
{channel.statistics && <ChannelStatistics statistics={channel.statistics}/>}
</div>
{channel && channel.statistics && <ChannelStatistics statistics={channel.statistics}/>}
</>
}
export function ChannelDownloadForm({channel_id, afterSave, closeModal, download}) {
download = download || {};
channel_id = download ? download.channel_id || channel_id : channel_id;
const editing = download && !_.isEmpty(download);
const settings = download.settings ? download.settings : {};
const oldTagNames = download && download.settings && download.settings.tag_names ? download.settings.tag_names : [];
const [state, setState] = React.useState({
frequency: download.frequency ? download.frequency : 604800,
title_exclude: settings.title_exclude ? settings.title_exclude : '',
title_include: settings.title_include ? settings.title_include : '',
url: download ? download.url : '',
})
const [disabled, setDisabled] = React.useState(useWROLMode());
const [loading, setLoading] = React.useState(false);
const [urlValid, setUrlValid] = React.useState(true);
const [tagNames, setTagNames] = React.useState(oldTagNames);
const handleInputChange = (e, {name, value}) => {
if (e) {
e.preventDefault();
}
if (name === 'url') {
setUrlValid(validURL(value));
}
setState({...state, [name]: value});
}
const handleSubmit = async () => {
setLoading(true);
setDisabled(true);
try {
let response;
if (editing) {
response = await updateChannelDownload(
channel_id,
download.id,
state.url,
state.frequency,
state.title_include,
state.title_exclude,
tagNames,
);
} else {
response = await createChannelDownload(
channel_id,
state.url,
state.frequency,
state.title_include,
state.title_exclude,
tagNames,
);
}
if (!response.ok) {
throw 'Creating download failed';
}
if (afterSave) {
await afterSave();
}
} finally {
setLoading(false);
setDisabled(false);
}
}
const handleClose = (e) => {
if (e) e.preventDefault();
closeModal();
}
const handleDelete = async () => {
await deleteDownload(download.id);
if (afterSave) {
afterSave();
}
if (closeModal) {
closeModal();
}
};
const deleteDownloadButton = <APIButton
color='red'
floated='left'
onClick={handleDelete}
confirmContent='Are you sure you want to delete this download?'
confirmButton='Delete'
disabled={disabled}
obeyWROLMode={true}
>Delete</APIButton>;
return <Form onSubmit={handleSubmit}>
{loading && <Dimmer active><Loader/></Dimmer>}
<Grid columns={2} stackable>
<Grid.Row>
<Grid.Column width={12}>
<FormInput
label='URL'
name='url'
type='url'
value={state.url}
placeholder='https://example.com/videos'
onChange={handleInputChange}
error={!urlValid}
/>
</Grid.Column>
<Grid.Column width={4}>
<FormField>
<label>Download Frequency</label>
<Dropdown selection
name='frequency'
placeholder='Frequency'
value={state.frequency}
disabled={disabled}
options={frequencyOptions.slice(1)}
onChange={handleInputChange}
/>
</FormField>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<HelpHeader
headerSize='h4'
headerContent='Title Match Words'
popupContent='List of words, separated by commas, that Video titles must contain to be downloaded.'
popupPosition='bottom center'
/>
<FormInput
name="title_include"
type="text"
disabled={disabled}
placeholder='Shelter,Solar Power'
value={state.title_include}
onChange={handleInputChange}
/>
</Grid.Column>
<Grid.Column>
<HelpHeader
headerSize='h4'
headerContent='Title Exclusion Words'
popupContent='List of words, separated by commas, that may not appear in video titles to be downloaded.'
popupPosition='bottom center'
/>
<FormInput
name="title_exclude"
type="text"
disabled={disabled}
placeholder='Giveaway,Prize'
value={state.title_exclude}
onChange={handleInputChange}
/>
</Grid.Column>
</Grid.Row>
<Grid.Row columns={1}>
<Grid.Column>
<TagsSelector selectedTagNames={tagNames} onChange={(i, j) => setTagNames(i)}/>
</Grid.Column>
</Grid.Row>
<Grid.Row columns={1}>
<Grid.Column textAlign='right'>
{editing && deleteDownloadButton}
<Button onClick={handleClose}>Cancel</Button>
<Button color='violet' disabled={!urlValid}>Save</Button>
</Grid.Column>
</Grid.Row>
</Grid>
</Form>
}
export function ChannelEditPage(props) {
return <ChannelPage header="Edit Channel" {...props}/>;
}

157
app/src/components/Common.js
View file

@ -1,5 +1,7 @@
import React, {useContext, useEffect, useState} from "react";
import {
AccordionContent,
AccordionTitle,
BreadcrumbSection,
Button as SButton,
ButtonGroup,
@ -29,6 +31,7 @@ import {
} from "../hooks/customHooks";
import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/contexts";
import {
Accordion,
Breadcrumb,
BreadcrumbDivider,
Button,
@ -52,21 +55,7 @@ import {
import {FilePreviewContext} from "./FilePreview";
import _ from "lodash";
import {killDownloads, startDownloads} from "../api";
export const API_URI = process.env && process.env.REACT_APP_API_URI ? process.env.REACT_APP_API_URI : `https://${window.location.host}/api`;
export const VIDEOS_API = `${API_URI}/videos`;
export const ARCHIVES_API = `${API_URI}/archive`;
export const OTP_API = `${API_URI}/otp`;
export const ZIM_API = `${API_URI}/zim`;
export const DEFAULT_LIMIT = 20;
export const NAME = process.env && process.env.REACT_APP_NAME ? process.env.REACT_APP_NAME : null;
// Other services on a WROLPi.
const port = window.location.port ? `${window.location.port}` : '';
export const FILES_MEDIA_URI = `https://${window.location.hostname}${port}/media/`;
export const MAP_VIEWER_URI = `https://${window.location.hostname}:8084`;
export const ZIM_VIEWER_URI = `https://${window.location.hostname}:8085`;
export const HELP_VIEWER_URI = `https://${window.location.hostname}:8086`;
import {allFrequencyOptions, NAME, semanticUIColorMap, validUrlRegex} from "./Vars";
export function Paginator({activePage, onPageChange, totalPages, showFirstAndLast, size = 'mini'}) {
const handlePageChange = (e, {activePage}) => {
@ -105,7 +94,7 @@ export function divmod(x, y) {
return [Math.floor(x / y), x % y];
}
export function secondsElapsed(seconds, short = true) {
export function secondsToHumanElapsed(seconds, short = true) {
// Convert the provided seconds into a human-readable string of the time elapsed between the provided timestamp
// and now.
if (!seconds || seconds < 0) {
@ -158,7 +147,7 @@ export function secondsElapsed(seconds, short = true) {
export function secondsToElapsedPopup(seconds) {
// Return a Popup which allows the user see a more detailed timestamp when hovering.
const elapsed = secondsElapsed(seconds);
const elapsed = secondsToHumanElapsed(seconds);
if (!elapsed) {
return <></>;
}
@ -177,7 +166,7 @@ export function isoDatetimeToElapsedPopup(dt) {
export function isoDatetimeToAgoPopup(dt, short = true) {
const seconds = (new Date(dt)).getTime() / 1000;
// Return a Popup which allows the user see a more detailed timestamp when hovering.
const elapsed = secondsElapsed(seconds, short);
const elapsed = secondsToHumanElapsed(seconds, short);
if (seconds === 0 || !elapsed) {
return <></>;
}
@ -271,31 +260,9 @@ export function RequiredAsterisk() {
return <span style={{color: '#db2828'}}> *</span>
}
export let defaultFileOrder = '-published_datetime';
export let defaultSearchOrder = 'rank';
export const frequencyOptions = [{key: null, text: '', value: null}, {
key: 'daily', text: 'Daily', value: 86400
}, {key: 'weekly', text: 'Weekly', value: 604800}, {key: 'biweekly', text: 'Biweekly', value: 1209600}, {
key: '30days', text: '30 Days', value: 2592000
}, {key: '90days', text: '90 Days', value: 7776000},];
export const rssFrequencyOptions = [{key: 'once', text: 'Once', value: 0}, {
key: 'hourly', text: 'Hourly', value: 3600
}, {key: '3hours', text: '3 hours', value: 10800}, {key: '12hours', text: '12 hours', value: 43200}, {
key: 'daily', text: 'Daily', value: 86400
}, {key: 'weekly', text: 'Weekly', value: 604800}, {key: 'biweekly', text: 'Biweekly', value: 1209600}, {
key: '30days', text: '30 Days', value: 2592000
}, {key: '90days', text: '90 Days', value: 7776000}, {key: '180days', text: '180 Days', value: 15552000}];
export function secondsToFrequency(seconds) {
for (let i = 0; i < Object.keys(rssFrequencyOptions).length; i++) {
let d = rssFrequencyOptions[i];
if (d.value === seconds) {
return d.text;
}
}
return null;
const option = allFrequencyOptions[seconds];
return option ? option.text : null;
}
const secondsToYears = 31536000;
@ -782,10 +749,14 @@ export function HelpHeader({
iconSize,
headerContent,
popupContent,
popupPosition = 'top center'
popupPosition = null,
for_ = null,
required = false,
}) {
return <div className='inline-header'>
<Header as={headerSize}>{headerContent}</Header>
<label for={for_}>
<Header as={headerSize}>{headerContent} {required && <RequiredAsterisk/>}</Header>
</label>
<span>
<HelpPopup content={popupContent} size={iconSize} icon={icon} position={popupPosition}/>
</span>
@ -1205,7 +1176,7 @@ export function useTitle(title) {
}, [title]);
}
export function DirectorySearch({onSelect, value, ...props}) {
export function DirectorySearch({onSelect, value, disabled, required, ...props}) {
const {
directoryName,
setDirectoryName,
@ -1280,6 +1251,7 @@ export function DirectorySearch({onSelect, value, ...props}) {
loading={loading}
value={directoryName}
results={results}
disabled={disabled}
{...props}
/>
}
@ -1289,21 +1261,6 @@ export const BackButton = ({...props}) => {
return <Button icon='arrow left' content='Back' onClick={() => navigate(-1)} {...props}/>;
}
export const semanticUIColorMap = {
red: '#db2828',
orange: '#f2711c',
yellow: '#fbbd08',
olive: '#b5cc18',
green: '#21ba45',
teal: '#00b5ad',
blue: '#2185d0',
violet: '#6435c9',
purple: '#a333c8',
pink: '#e03997',
brown: '#a5673f',
grey: '#767676',
}
export const ColorToSemanticHexColor = (color) => {
return semanticUIColorMap[color] || null;
}
@ -1456,13 +1413,15 @@ export function useAPIButton(
themed = true,
obeyWROLMode = false,
icon = null,
type = 'button',
id = null,
props
) {
props = props || {};
const ref = React.useRef();
const [confirmOpen, setConfirmOpen] = React.useState(false);
const [pending, setPending] = React.useState(false);
const [loading, setLoading] = React.useState(false);
const [animation, setAnimation] = React.useState('jiggle');
const [animationVisible, setAnimationVisible] = React.useState(true);
const [showSuccess, setShowSuccess] = React.useState(false);
@ -1471,7 +1430,7 @@ export function useAPIButton(
const wrolModeEnabled = useWROLMode();
// Disable when API call is pending, or button is disabled.
disabled = pending || disabled;
disabled = loading || disabled;
// Disable when WROL Mode is enabled, otherwise normal disabled.
disabled = obeyWROLMode ? wrolModeEnabled || disabled : disabled;
@ -1496,15 +1455,16 @@ export function useAPIButton(
const handleAPICall = async () => {
// Handle when user clicks button, or clicks confirm.
setPending(true);
setLoading(true);
try {
await onClick();
setSuccess();
} catch (e) {
console.error(e);
setFailure();
} finally {
setLoading(false);
}
setPending(false);
}
const localOnClick = async (e) => {
@ -1533,7 +1493,14 @@ export function useAPIButton(
}
// Create button with or without theme. Pass all props to the <Button/> (except props.children).
const buttonArgs = {color, onClick: localOnClick, disabled, loading: pending, size, floated, ...props};
const buttonArgs = {
color, onClick: localOnClick, disabled, loading, size, floated, type,
...props
};
if (id) {
buttonArgs['id'] = id;
}
let buttonContent = props.children || null;
if (icon) {
@ -1587,6 +1554,8 @@ export function APIButton({
themed,
obeyWROLMode,
icon,
type = 'button',
id = null,
...props
}) {
const {button} = useAPIButton(
@ -1601,6 +1570,8 @@ export function APIButton({
themed,
obeyWROLMode,
icon,
type,
id,
props
);
@ -1811,12 +1782,24 @@ export function Breadcrumbs({crumbs, size = undefined}) {
</Breadcrumb>
}
const validUrlRegex = /^(http|https):\/\/[^ "]+$/;
export function validURL(url) {
return !(url && !validUrlRegex.test(url));
}
export function validURLs(urls) {
if (!!!urls) {
// Invalid while empty.
return false;
}
urls = urls.split(/\r?\n/);
for (let i = 0; i < urls.length; i++) {
if (!validURL(urls[i])) {
return false;
}
}
return true;
}
export function useLocalStorage(key, initialValue, decode = JSON.parse, encode = JSON.stringify) {
// Use localstorage to store some JSON encode-able.
@ -1852,3 +1835,43 @@ export function useLocalStorageInt(key, initialValue) {
const [storedValue, setStoredValue] = useLocalStorage(key, initialValue, parseInt, (num) => num.toString());
return [storedValue, setStoredValue];
}
export function SimpleAccordion({title = 'Advanced', ...props}) {
const [active, setActive] = React.useState(false);
return <Accordion>
<AccordionTitle
active={active}
onClick={() => setActive(!active)}
>
<Icon name='dropdown'/>
{title}
</AccordionTitle>
<AccordionContent active={active}>
{props.children}
</AccordionContent>
</Accordion>
}
export function mergeDeep(target, source) {
if (_.isEmpty(source)) {
return target;
}
// Initialize the result as target
let result = Object.assign({}, target);
for (let key of Object.keys(source)) {
if (Array.isArray(source[key])) {
// source overwrites target if it is a list with values.
result[key] = source[key] || target[key];
} else if (typeof source[key] === 'object' && typeof target[key] === 'object') {
result[key] = mergeDeep(target[key] || {}, source[key]);
} else {
result[key] = source[key] !== undefined ? source[key] : target[key];
}
}
return result;
}

100
app/src/components/Common.test.js
View file

@ -1,100 +0,0 @@
import React from "react";
import {getByTestId, queryByTestId, render, screen} from '@testing-library/react';
import {CPUTemperatureIcon, DisableDownloadsToggle} from "./Common";
import {StatusContext} from "../contexts/contexts";
import {startDownloads} from "../components/Common";
import {BrowserRouter} from "react-router-dom";
const mockedUseHref = jest.fn();
jest.mock("react-router-dom", () => ({
...(jest.requireActual("react-router-dom")),
useHref: () => mockedUseHref
}));
test('DownloadsToggle is off when loading', async () => {
const statusValue = {status: null, fetchStatus: null};
const {container} = render(<StatusContext.Provider value={statusValue}>
<DisableDownloadsToggle/>
</StatusContext.Provider>);
let toggle = screen.getByRole('checkbox');
expect(toggle).not.toBeChecked();
// Disabled while loading.
expect(toggle).toHaveClass('disabled');
// Text is "disabled" when downloading is On.
const label = getByTestId(container, 'toggle-label');
expect(label).toHaveTextContent('Downloading Disabled');
});
test('DownloadsToggle is off if downloading is Off', async () => {
const statusValue = {
status: {downloads: {disabled: true, stopped: true}},
fetchStatus: null,
}
const {container} = render(<StatusContext.Provider value={statusValue}>
<DisableDownloadsToggle/>
</StatusContext.Provider>);
const toggle = screen.getByRole('checkbox');
expect(toggle).not.toBeChecked();
// Not disabled when loaded.
expect(toggle).not.toHaveClass('disabled');
// Text is "disabled" when downloading is On.
const label = getByTestId(container, 'toggle-label');
expect(label).toHaveTextContent('Downloading Disabled');
});
test('DownloadsToggle is on if downloading is On', async () => {
const statusValue = {
status: {downloads: {disabled: false, stopped: false}},
fetchStatus: null,
}
const {container} = render(<StatusContext.Provider value={statusValue}>
<DisableDownloadsToggle/>
</StatusContext.Provider>);
const toggle = screen.getByRole('checkbox');
expect(toggle).toBeChecked();
// Not disabled when loaded.
expect(toggle).not.toHaveClass('disabled');
// Text is "enabled" when downloading is On.
const label = getByTestId(container, 'toggle-label');
expect(label).toHaveTextContent('Downloading Enabled');
});
test('CPUTemperatureIcon high', async () => {
const statusValue = {status: {cpu_info: {temperature: 80, high_temperature: 75, critical_temperature: 90}}};
const {container} = render(<StatusContext.Provider value={statusValue}>
<BrowserRouter>
<CPUTemperatureIcon/>
</BrowserRouter>
</StatusContext.Provider>);
const icon = getByTestId(container, 'cpuTemperatureIcon');
expect(icon).toHaveClass('thermometer');
expect(icon).toHaveClass('half');
});
test('CPUTemperatureIcon critical', async () => {
const statusValue = {status: {cpu_info: {temperature: 100, high_temperature: 75, critical_temperature: 90}}};
const {container} = render(<StatusContext.Provider value={statusValue}>
<BrowserRouter>
<CPUTemperatureIcon/>
</BrowserRouter>
</StatusContext.Provider>);
const icon = getByTestId(container, 'cpuTemperatureIcon');
expect(icon).toHaveClass('thermometer');
expect(icon).not.toHaveClass('half');
});
test('CPUTemperatureIcon hidden', async () => {
const statusValue = {status: {cpu_info: {temperature: 35, high_temperature: 75, critical_temperature: 90}}};
const {container} = render(<StatusContext.Provider value={statusValue}>
<CPUTemperatureIcon/>
</StatusContext.Provider>);
const icon = queryByTestId(container, 'cpuTemperatureIcon');
expect(icon).toBeNull();
});

68
app/src/components/Download.cy.js Normal file
View file

@ -0,0 +1,68 @@
import React from 'react';
import {ArchiveDownloadForm, ChannelDownloadForm} from './Download';
import 'semantic-ui-offline/semantic.min.css';
describe('<ArchiveDownloadForm />', () => {
beforeEach(() => {
cy.mountWithTags(<ArchiveDownloadForm/>);
});
it('displays error for invalid URL', () => {
cy.get('#urls_textarea').type('invalid-url').wait(500);
cy.get('#urls_textarea').should('have.attr', 'error');
});
it('enables download button for valid URL', () => {
cy.get('#urls_textarea').type('https://wrolpi.org').wait(500);
cy.get('#urls_textarea').should('not.have.attr', 'error');
cy.get('#download_form_download_button').should('be.visible').and('not.have.attr', 'disabled');
});
it('disables download button for invalid URL', () => {
cy.get('#urls_textarea').type('invalid-url').wait(500);
cy.get('#urls_textarea').should('have.attr', 'error');
cy.get('#download_form_download_button').should('be.visible').and('have.attr', 'disabled');
});
it('initially has download button disabled', () => {
cy.get('#download_form_download_button').should('be.visible').and('have.attr', 'disabled');
});
it('clears error when switching from invalid to valid URL', () => {
cy.get('#urls_textarea').type('invalid-url').wait(500);
cy.get('#urls_textarea').should('have.attr', 'error');
cy.get('#urls_textarea').clear();
cy.get('#urls_textarea').type('https://wrolpi.org').wait(500);
cy.get('#urls_textarea').should('not.have.attr', 'error');
});
});
describe('<ChannelDownloadForm/>', () => {
beforeEach(() => {
// Do not return directories for these tests.
cy.intercept('POST', '**/api/files/search_directories', {
statusCode: 200,
body: {
is_dir: true, channel_directories: [], domain_directories: [],
directories: [{path: "some directory", name: "some directory"}],
},
}).as('searchDirectories');
cy.mountWithTags(<ChannelDownloadForm/>);
});
it('displays error for invalid URL', () => {
cy.get('#url_input').type('invalid-url').wait(500);
cy.get('#url_input').should('have.attr', 'error');
});
it('Can choose different options', () => {
cy.get('#download_frequency_selector').click()
.get('.item').contains('Once').click();
cy.get('#download_frequency_selector').click()
.get('.item').contains('Daily').click();
cy.get('#download_frequency_selector').click()
.get('.item').contains('180 Days').click();
});
});

1653
app/src/components/Download.js
View file

File diff suppressed because it is too large Load diff

2
app/src/components/Files.js
View file

@ -20,7 +20,6 @@ import {
ErrorMessage,
ExternalCardLink,
FileIcon,
FILES_MEDIA_URI,
findPosterPath,
HandPointMessage,
humanFileSize,
@ -69,6 +68,7 @@ import {useSubscribeEventName} from "../Events";
import {TagsSelector} from "../Tags";
import {Headlines} from "./Headline";
import {useSearch} from "./Search";
import {FILES_MEDIA_URI} from "./Vars";
function EbookCard({file}) {
const {s} = useContext(ThemeContext);

1
app/src/components/Inventory.js
View file

@ -29,7 +29,6 @@ import _ from 'lodash';
import {ThemeContext} from "../contexts/contexts";
import {useInventory} from "../hooks/customHooks";
import {Button, FormGroup, FormInput, Header, Segment, Table} from "./Theme";
import {toast} from "react-semantic-toasts-2";
function InventorySummary() {
const {t} = useContext(ThemeContext);

2
app/src/components/Map.js
View file

@ -7,7 +7,6 @@ import {
humanFileSize,
IframeViewer,
InfoMessage,
MAP_VIEWER_URI,
PageContainer,
TabLinks,
useTitle,
@ -32,6 +31,7 @@ import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {Header, Loader, Placeholder, Segment, Table} from "./Theme";
import {StatusContext} from "../contexts/contexts";
import _ from "lodash";
import {MAP_VIEWER_URI} from "./Vars";
function DockerMapImportWarning() {
const {status} = useContext(StatusContext);

3
app/src/components/Nav.js
View file

@ -5,9 +5,7 @@ import {Media, SettingsContext, StatusContext, ThemeContext} from "../contexts/c
import {
CPUTemperatureIcon,
DarkModeToggle,
HELP_VIEWER_URI,
HotspotStatusIcon,
NAME,
SystemLoadIcon,
useLocalStorage
} from "./Common";
@ -15,6 +13,7 @@ import {ShareButton} from "./Share";
import {useWROLMode} from "../hooks/customHooks";
import {SearchIconButton} from "./Search";
import {Icon} from "./Theme";
import {HELP_VIEWER_URI, NAME} from "./Vars";
const links = [
{text: 'Videos', to: '/videos', key: 'videos'},

127
app/src/components/Vars.js Normal file
View file

@ -0,0 +1,127 @@
export const API_URI = process.env && process.env.REACT_APP_API_URI ? process.env.REACT_APP_API_URI : `https://${window.location.host}/api`;
export const VIDEOS_API = `${API_URI}/videos`;
export const ARCHIVES_API = `${API_URI}/archive`;
export const OTP_API = `${API_URI}/otp`;
export const ZIM_API = `${API_URI}/zim`;
export const DEFAULT_LIMIT = 20;
export const NAME = process.env && process.env.REACT_APP_NAME ? process.env.REACT_APP_NAME : null;
// Other services on a WROLPi.
const port = window.location.port ? `${window.location.port}` : '';
export const FILES_MEDIA_URI = `https://${window.location.hostname}${port}/media/`;
export const MAP_VIEWER_URI = `https://${window.location.hostname}:8084`;
export const ZIM_VIEWER_URI = `https://${window.location.hostname}:8085`;
export const HELP_VIEWER_URI = `https://${window.location.hostname}:8086`;
export let defaultFileOrder = '-published_datetime'; // Most recently published files first.
export let defaultSearchOrder = 'rank'; // Most relevant files first.
// Download frequencies.
export const nullOption = {key: null, text: '', value: null};
export const onceOption = {key: 0, text: 'Once', value: 0};
export const hourlyOption = {key: 3600, text: 'Hourly', value: 3600};
export const threeHoursOption = {key: 10800, text: '3 hours', value: 10800};
export const twelveHoursOption = {key: 43200, text: '12 hours', value: 43200};
export const dailyOption = {key: 86400, text: 'Daily', value: 86400};
export const weeklyOption = {key: 604800, text: 'Weekly', value: 604800};
export const biweeklyOption = {key: 1209600, text: 'Biweekly', value: 1209600};
export const days30Option = {key: 2592000, text: '30 Days', value: 2592000};
export const days90Option = {key: 7776000, text: '90 Days', value: 7776000};
export const days180Option = {key: 15552000, text: '180 Days', value: 15552000};
export const frequencyOptions = [
dailyOption,
weeklyOption,
biweeklyOption,
days30Option,
days90Option,
];
export const longFrequencyOptions = [
dailyOption,
weeklyOption,
biweeklyOption,
days30Option,
days90Option,
days180Option,
];
export const channelFrequencyOptions = [
onceOption, // A Channel/Playlist can be downloaded once.
...longFrequencyOptions,
]
export const extendedFrequencyOptions = [
onceOption,
hourlyOption,
threeHoursOption,
twelveHoursOption,
...longFrequencyOptions,
];
export const allFrequencyOptions = {
[nullOption.value]: nullOption,
[onceOption.value]: onceOption,
[hourlyOption.value]: hourlyOption,
[threeHoursOption.value]: threeHoursOption,
[twelveHoursOption.value]: twelveHoursOption,
[dailyOption.value]: dailyOption,
[weeklyOption.value]: weeklyOption,
[biweeklyOption.value]: biweeklyOption,
[days30Option.value]: days30Option,
[days90Option.value]: days90Option,
[days180Option.value]: days180Option,
};
export const semanticUIColorMap = {
red: '#db2828',
orange: '#f2711c',
yellow: '#fbbd08',
olive: '#b5cc18',
green: '#21ba45',
teal: '#00b5ad',
blue: '#2185d0',
violet: '#6435c9',
purple: '#a333c8',
pink: '#e03997',
brown: '#a5673f',
grey: '#767676',
}
export const validUrlRegex = /^(http|https):\/\/[^ "]+$/;
export const downloadOrderOptions = [
{key: 'newest', text: 'Newest', value: 'newest'},
{key: 'oldest', text: 'Oldest', value: 'oldest'},
{key: 'views', text: 'Most Views', value: 'views'},
];
export const downloadResolutionOptions = [
{key: '360p', text: '360p', value: '360p'},
{key: '480p', text: '480p', value: '480p'},
{key: '720p', text: '720p', value: '720p'},
{key: '1080p', text: '1080p', value: '1080p'},
{key: '1440p', text: '1440p', value: '1440p'},
{key: '2160p', text: '2160p', value: '2160p'},
{key: 'maximum', text: 'Maximum', value: 'maximum'},
]
export const defaultVideoResolutionOptions = ['1080p', '720p', '480p', 'maximum'];
export const downloadFormatOptions = [
{key: 'mp4', text: '.mp4', value: 'mp4'},
{key: 'mkv', text: '.mkv', value: 'mkv'},
]
export const defaultVideoFormatOption = 'mp4';
export const Downloaders = {
Archive: 'archive',
File: 'file',
KiwixCatalog: 'kiwix_catalog',
KiwixZim: 'kiwix_zim',
RSS: 'rss',
ScrapeHtml: 'scrape_html',
Video: 'video',
VideoChannel: 'video_channel',
};

8
app/src/components/VideoPlayer.js
View file

@ -8,8 +8,8 @@ import {
encodeMediaPath,
getParentDirectory,
humanFileSize,
humanNumber, isoDatetimeToAgoPopup,
isoDatetimeToString,
humanNumber,
isoDatetimeToAgoPopup,
MultilineText,
PageContainer,
PreviewPath,
@ -110,7 +110,7 @@ const VideoComment = ({comment, children}) => {
specialIcon = <Icon name='heart' color='red'/>;
}
const dateElm = <div {...t}>{isoDatetimeToString(timestamp * 1000)}</div>;
const dateElm = <div {...t}>{isoDatetimeToAgoPopup(timestamp * 1000)}</div>;
const likesElm = like_count && <div {...t}>
<Icon name='thumbs up'/>{humanNumber(comment['like_count'])}
</div>;
@ -303,7 +303,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
</Grid.Column>
<Grid.Column>
<h3>Censored</h3>
<p>{video.censored ? 'Yes' : 'No'}</p>
<p>{videoFile.censored ? 'Yes' : 'No'}</p>
</Grid.Column>
</Grid.Row>
<Grid.Row columns={1}>

123
app/src/components/Videos.js
View file

@ -4,8 +4,6 @@ import {
APIButton,
CardLink,
CardPoster,
defaultFileOrder,
defaultSearchOrder,
Duration,
encodeMediaPath,
ErrorMessage,
@ -14,7 +12,6 @@ import {
humanFileSize,
humanNumber,
isoDatetimeToAgoPopup,
isoDatetimeToString,
mimetypeColor,
PageContainer,
PreviewLink,
@ -32,6 +29,7 @@ import {
CardDescription,
CardHeader,
Container,
Form,
Image,
PlaceholderHeader,
PlaceholderLine,
@ -45,9 +43,12 @@ import {FileRowTagIcon, FilesView} from "./Files";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Icon from "semantic-ui-react/dist/commonjs/elements/Icon";
import {Button, Card, Header, Loader, Placeholder, Popup, Segment, Statistic, StatisticGroup} from "./Theme";
import {deleteVideos} from "../api";
import {deleteVideos, fetchVideoDownloaderConfig, saveVideoDownloaderConfig} from "../api";
import {Media, QueryContext, ThemeContext} from "../contexts/contexts";
import _ from "lodash";
import {defaultFileOrder, defaultSearchOrder} from "./Vars";
import {InputForm, ToggleForm, useForm} from "../hooks/useForm";
import {VideoResolutionSelectorForm} from "./Download";
export function VideoWrapper() {
const {videoId} = useParams();
@ -221,6 +222,114 @@ function VideosPage() {
</>
}
function VideosSettings() {
useTitle('Videos Settings');
const emptyFormData = {
video_resolutions: ['1080p', '720p', '480p', 'maximum'],
yt_dlp_options: {
file_name_format: '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s',
nooverwrites: true,
writeautomaticsub: true,
writesubtitles: true,
writethumbnail: true,
},
};
const fetcher = async () => {
return await fetchVideoDownloaderConfig();
}
const submitter = async () => {
return await saveVideoDownloaderConfig(form.formData);
}
const form = useForm({
fetcher,
submitter,
emptyFormData,
});
return <Segment>
<Header as='h3'>Video Downloader Settings</Header>
<Form>
<Grid>
<Grid.Row>
<Grid.Column>
<VideoResolutionSelectorForm
form={form}
name='video_resolutions'
path='video_resolutions'
/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<InputForm
form={form}
name='file_name_format'
path='yt_dlp_options.file_name_format'
label='Video File Format'
/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<ToggleForm
form={form}
label='Do not overwrite existing files'
name='nooverwrites'
path='yt_dlp_options.nooverwrites'
/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<ToggleForm
form={form}
label='Download automatic subtitles'
name='writeautomaticsub'
path='yt_dlp_options.writeautomaticsub'
/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<ToggleForm
form={form}
label='Download subtitles'
name='writesubtitles'
path='yt_dlp_options.writesubtitles'
/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<ToggleForm
form={form}
label='Download thumbnail'
name='writethumbnail'
path='yt_dlp_options.writethumbnail'
/>
</Grid.Column>
</Grid.Row>
<Grid.Row columns={1}>
<Grid.Column textAlign='right'>
<APIButton
disabled={form.disabled || !form.ready}
type='submit'
style={{marginTop: '0.5em'}}
onClick={form.onSubmit}
id='video_settings_save_button'
>Save</APIButton>
</Grid.Column>
</Grid.Row>
</Grid>
</Form>
</Segment>
}
function VideosStatistics() {
useTitle('Video Statistics');
@ -286,6 +395,7 @@ export function VideosRoute(props) {
const links = [
{text: 'Videos', to: '/videos', key: 'videos', end: true},
{text: 'Channels', to: '/videos/channel', key: 'channel'},
{text: 'Settings', to: '/videos/settings', key: 'settings'},
{text: 'Statistics', to: '/videos/statistics', key: 'statistics'},
];
@ -294,6 +404,7 @@ export function VideosRoute(props) {
<Routes>
<Route path='/' exact element={<VideosPage/>}/>
<Route path='channel' exact element={<ChannelsPage/>}/>
<Route path='settings' exact element={<VideosSettings/>}/>
<Route path='statistics' exact element={<VideosStatistics/>}/>
<Route path='channel/new' exact element={<ChannelNewPage/>}/>
<Route path='channel/:channelId/edit' exact element={<ChannelEditPage/>}/>
@ -384,7 +495,7 @@ export function VideoRowCells({file}) {
poster = <FileIcon file={file} size='large'/>;
}
let dataCell = file.published_datetime ? isoDatetimeToString(file.published_datetime) : '';
let dataCell = file.published_datetime ? isoDatetimeToAgoPopup(file.published_datetime) : '';
if (sort === 'length') {
dataCell = secondsToFullDuration(file.length || 0);
} else if (sort === 'size') {
@ -392,7 +503,7 @@ export function VideoRowCells({file}) {
} else if (sort === 'view_count') {
dataCell = humanNumber(video.view_count || 0);
} else if (sort === 'viewed') {
dataCell = isoDatetimeToString(file.viewed);
dataCell = isoDatetimeToAgoPopup(file.viewed);
}
// Fragment for SelectableRow

4
app/src/components/Zim.js
View file

@ -43,8 +43,7 @@ import {
TabLinks,
TagIcon,
useTitle,
WarningMessage,
ZIM_VIEWER_URI
WarningMessage
} from "./Common";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import {TagsSelector} from "../Tags";
@ -66,6 +65,7 @@ import {Link, Route, Routes} from "react-router-dom";
import {SortableTable} from "./SortableTable";
import {toast} from "react-semantic-toasts-2";
import _ from "lodash";
import {ZIM_VIEWER_URI} from "./Vars";
export const OutdatedZimsMessage = ({onClick}) => {
const [open, setOpen] = React.useState(false);

16
app/src/components/admin/Configs.js
View file

@ -1,17 +1,7 @@
import {useConfigs} from "../../hooks/customHooks";
import {APIButton, PageContainer} from "../Common";
import {Header, Icon, Popup, Table} from "../Theme";
import {APIButton} from "../Common";
import {Icon, Table} from "../Theme";
import React from "react";
import {
Grid,
GridColumn,
GridRow,
TableBody,
TableCell,
TableHeader,
TableHeaderCell,
TableRow
} from "semantic-ui-react";
import {TableBody, TableCell, TableHeader, TableHeaderCell, TableRow} from "semantic-ui-react";
import {TableRowPlaceholder} from "../Placeholder";
import {useNavigate} from "react-router-dom";

414
app/src/components/admin/Downloads.js
View file

@ -4,7 +4,6 @@ import {
deleteDownload,
deleteOnceDownloads,
killDownload,
putDownload,
restartDownload,
retryOnceDownloads
} from "../../api";
@ -13,18 +12,13 @@ import {
APIButton,
DisableDownloadsToggle,
ErrorMessage,
frequencyOptions,
isoDatetimeToElapsedPopup,
secondsToFrequency,
useTitle,
validURL,
WROLModeMessage
} from "../Common";
import {
Button as SButton,
Dimmer,
Dropdown,
Grid,
Label,
Loader,
PlaceholderLine,
@ -35,23 +29,10 @@ import {
TableHeaderCell,
TableRow
} from "semantic-ui-react";
import {
Button,
Form,
FormField,
FormInput,
Header,
Modal,
ModalActions,
ModalContent,
ModalHeader,
Placeholder,
Segment,
Table
} from "../Theme";
import {useDownloads, useWROLMode} from "../../hooks/customHooks";
import {toast} from "react-semantic-toasts-2";
import {ChannelDownloadForm} from "../Channels";
import {Button, Header, Modal, ModalActions, ModalContent, ModalHeader, Placeholder, Segment, Table} from "../Theme";
import {useDownloads} from "../../hooks/customHooks";
import {EditChannelDownloadForm, EditRSSDownloadForm, EditZimDownloadForm} from "../Download";
import {Downloaders} from "../Vars";
function ClearDownloadsButton({callback}) {
async function localClearDownloads() {
@ -112,251 +93,142 @@ function DeleteOnceDownloadsButton({callback}) {
}
export function DownloadEditForm({afterSave, closeModal, download}) {
const [state, setState] = React.useState({
url: download.url || '',
frequency: download.frequency,
excluded_urls: download.settings ? download.settings.excluded_urls : '',
destination: download.settings ? download.settings.destination : '',
})
const [disabled, setDisabled] = React.useState(useWROLMode());
const [loading, setLoading] = React.useState(false);
const [urlValid, setUrlValid] = React.useState(true);
function RecurringDownloadRow({download, fetchDownloads, onDelete}) {
const [errorModalOpen, setErrorModalOpen] = React.useState(false);
const [editModalOpen, setEditModalOpen] = React.useState(false);
const handleInputChange = (e, {name, value}) => {
if (e) {
e.preventDefault();
}
if (name === 'url') {
setUrlValid(validURL(value));
}
setState({...state, [name]: value});
}
const handleSubmit = async () => {
setLoading(true);
setDisabled(true);
try {
const response = await putDownload(
[state.url],
download.id,
download.downloader,
download.sub_downloader,
state.frequency,
state.excluded_urls,
);
if (!response.ok) {
throw 'Updating download failed';
}
if (afterSave) {
afterSave();
}
} finally {
setLoading(false);
setDisabled(false);
}
}
const handleClose = (e) => {
if (e) e.preventDefault();
closeModal();
}
const handleDelete = async () => {
await deleteDownload(download.id);
if (afterSave) {
afterSave();
}
if (closeModal) {
closeModal();
}
};
const deleteDownloadButton = <APIButton
color='red'
floated='left'
onClick={handleDelete}
confirmContent='Are you sure you want to delete this download?'
confirmButton='Delete'
disabled={disabled}
obeyWROLMode={true}
>Delete</APIButton>;
return <Form onSubmit={handleSubmit}>
{loading && <Dimmer active><Loader/></Dimmer>}
<Grid columns={2} stackable>
<Grid.Row>
<Grid.Column width={12}>
<FormInput
label='URL'
name='url'
type='url'
value={state.url}
placeholder='https://example.com/videos'
onChange={handleInputChange}
error={!urlValid}
/>
</Grid.Column>
<Grid.Column width={4}>
<FormField>
<label>Download Frequency</label>
<Dropdown selection
name='frequency'
placeholder='Frequency'
value={state.frequency}
disabled={disabled}
options={frequencyOptions.slice(1)}
onChange={handleInputChange}
/>
</FormField>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<FormInput
label='Excluded URLs'
name='excluded_urls'
type='text'
value={state.excluded_urls}
onChange={handleInputChange}
/>
</Grid.Column>
<Grid.Column>
<FormInput
label='Destination'
name='destination'
type='text'
value={state.destination}
onChange={handleInputChange}
/>
</Grid.Column>
</Grid.Row>
<Grid.Row columns={1}>
<Grid.Column textAlign='right'>
{deleteDownloadButton}
<Button onClick={handleClose}>Cancel</Button>
<Button color='violet' disabled={!urlValid}>Save</Button>
</Grid.Column>
</Grid.Row>
</Grid>
</Form>
}
class RecurringDownloadRow extends React.Component {
constructor(props) {
super(props);
this.state = {
errorModalOpen: false,
editModalOpen: false,
}
}
handleRestart = async () => {
const {id} = this.props;
const handleRestart = async () => {
const {id} = download;
try {
await restartDownload(id);
} finally {
if (this.props.fetchDownloads) {
this.props.fetchDownloads();
if (fetchDownloads) {
await fetchDownloads();
}
}
}
render() {
let {
url,
frequency,
last_successful_download,
status,
location,
next_download,
error,
downloader,
} = this.props;
const {errorModalOpen} = this.state;
const handleEditOpen = () => setEditModalOpen(true);
const handleEditClose = () => setEditModalOpen(false);
const handleErrorOpen = () => setErrorModalOpen(true);
const handleErrorClose = () => setErrorModalOpen(false);
const link = location ?
(text) => <Link to={location}>{text}</Link> :
(text) => <a href={url} target='_blank' rel='noopener noreferrer'>{text}</a>;
let {
url,
frequency,
last_successful_download,
status,
location,
next_download,
error,
downloader,
} = download;
const errorModal = <Modal
closeIcon
onClose={() => this.setState({errorModalOpen: false})}
onOpen={() => this.setState({errorModalOpen: true})}
open={errorModalOpen}
trigger={<Button icon='exclamation circle' color='orange'/>}
>
<ModalHeader>Download Error</ModalHeader>
<ModalContent>
<pre style={{overflowX: 'scroll'}}>{error}</pre>
</ModalContent>
<ModalActions>
<SButton onClick={() => this.setState({errorModalOpen: false})}>Close</SButton>
</ModalActions>
</Modal>;
const link = location ?
(text) => <Link to={location}>{text}</Link> :
(text) => <a href={url} target='_blank' rel='noopener noreferrer'>{text}</a>;
const editButton = <>
<Button icon='edit' onClick={() => this.setState({editModalOpen: true})}/>
</>
const errorModal = <Modal
closeIcon
onClose={handleErrorClose}
onOpen={handleErrorOpen}
open={errorModalOpen}
trigger={<Button icon='exclamation circle' color='orange'/>}
>
<ModalHeader>Download Error</ModalHeader>
<ModalContent>
<pre style={{overflowX: 'scroll'}}>{error}</pre>
</ModalContent>
<ModalActions>
<SButton onClick={handleEditClose}>Close</SButton>
</ModalActions>
</Modal>;
const restartButton = <>
<APIButton
color='green'
icon='redo'
confirmContent='Are you sure you want to restart this download?'
confirmButton='Restart'
onClick={this.handleRestart}
obeyWROLMode={true}
/>
</>;
const editButton = <Button icon='edit' onClick={handleEditOpen}/>;
// Show "now" if we have passed the next_download.
let next = 'now';
if (next_download && new Date() < new Date(next_download)) {
next = isoDatetimeToElapsedPopup(next_download);
}
const restartButton = <APIButton
color='green'
icon='redo'
confirmContent='Are you sure you want to restart this download?'
confirmButton='Restart'
onClick={handleRestart}
obeyWROLMode={true}
/>;
const handleClose = () => this.setState({editModalOpen: false});
const afterSave = () => {
if (this.props.fetchDownloads) {
this.props.fetchDownloads();
}
handleClose();
}
const editModal = <Modal closeIcon
open={this.state.editModalOpen}
onClose={handleClose}
>
<ModalHeader>Edit Download</ModalHeader>
<ModalContent>
{downloader === 'video_channel' ?
<ChannelDownloadForm download={this.props} closeModal={handleClose}
afterSave={afterSave}/>
: <DownloadEditForm download={this.props} closeModal={handleClose}
afterSave={afterSave}/>
}
</ModalContent>
</Modal>;
return <TableRow>
<TableCell className='column-ellipsis'>
{link(url)}
</TableCell>
<TableCell>{secondsToFrequency(frequency)}</TableCell>
<TableCell>
{last_successful_download ? isoDatetimeToElapsedPopup(last_successful_download) : null}
{status === 'pending' ? <Loader active inline size='tiny'/> : null}
</TableCell>
<TableCell>{next}</TableCell>
<TableCell textAlign='right'>
{error && errorModal}
{editButton}
{restartButton}
</TableCell>
{editModal}
</TableRow>
// Show "now" if we have passed the next_download.
let next = 'now';
if (next_download && new Date() < new Date(next_download)) {
next = isoDatetimeToElapsedPopup(next_download);
}
const onSuccess = async () => {
if (fetchDownloads) {
await fetchDownloads();
}
handleEditClose();
}
const localOnDelete = async () => {
try {
await deleteDownload(download.id);
await onDelete();
} finally {
await fetchDownloads();
handleEditClose();
}
}
let editForm;
if (downloader === Downloaders.VideoChannel) {
editForm = <EditChannelDownloadForm
download={download}
onCancel={handleEditClose}
onSuccess={onSuccess}
onDelete={localOnDelete}
/>;
} else if (downloader === Downloaders.RSS) {
editForm = <EditRSSDownloadForm
download={download}
onDelete={localOnDelete}
onCancel={handleEditClose}
onSuccess={onSuccess}
/>;
} else if (downloader === Downloaders.KiwixCatalog) {
editForm = <EditZimDownloadForm
download={download}
onDelete={localOnDelete}
onCancel={handleEditClose}
onSuccess={onSuccess}
/>;
}
const editModal = <Modal closeIcon
open={editModalOpen}
onClose={handleEditClose}
>
<ModalHeader>Edit Download</ModalHeader>
<ModalContent>
{editForm}
</ModalContent>
</Modal>;
return <TableRow>
<TableCell className='column-ellipsis'>
{link(url)}
</TableCell>
<TableCell>{secondsToFrequency(frequency)}</TableCell>
<TableCell>
{last_successful_download && isoDatetimeToElapsedPopup(last_successful_download)}
{status === 'pending' && <Loader active inline size='tiny'/>}
</TableCell>
<TableCell>{next}</TableCell>
<TableCell textAlign='right'>
{error && errorModal}
{editButton}
{restartButton}
</TableCell>
{editModal}
</TableRow>
}
class OnceDownloadRow extends React.Component {
@ -365,28 +237,27 @@ class OnceDownloadRow extends React.Component {
}
handleDelete = async () => {
await deleteDownload(this.props.id);
await this.props.fetchDownloads();
try {
await deleteDownload(this.props.id);
} finally {
await this.props.fetchDownloads();
}
};
handleStop = async () => {
await killDownload(this.props.id);
await this.props.fetchDownloads();
try {
await killDownload(this.props.id);
} finally {
await this.props.fetchDownloads();
}
};
handleRestart = async () => {
try {
await restartDownload(this.props.id);
} catch (e) {
toast({
type: 'error',
title: 'Error',
description: 'Unable to restart download',
time: 5000,
})
throw e;
} finally {
await this.props.fetchDownloads();
}
await this.props.fetchDownloads();
};
render() {
@ -502,7 +373,7 @@ export function OnceDownloadsTable({downloads, fetchDownloads}) {
</Placeholder>
}
export function RecurringDownloadsTable({downloads, fetchDownloads}) {
export function RecurringDownloadsTable({downloads, fetchDownloads, onDelete}) {
if (downloads && downloads.length >= 1) {
return <Table compact className='table-ellipsis'>
<TableHeader>
@ -515,7 +386,14 @@ export function RecurringDownloadsTable({downloads, fetchDownloads}) {
</TableRow>
</TableHeader>
<TableBody>
{downloads.map(i => <RecurringDownloadRow key={i.id} fetchDownloads={fetchDownloads} {...i}/>)}
{downloads.map(i => {
return <RecurringDownloadRow
key={i.id}
fetchDownloads={fetchDownloads}
download={i}
onDelete={onDelete}
/>
})}
</TableBody>
</Table>
} else if (downloads) {

4
app/src/components/admin/Settings.js
View file

@ -21,7 +21,6 @@ import {
ErrorMessage,
HelpPopup,
HotspotToggle,
semanticUIColorMap,
ThrottleToggle,
Toggle,
WROLModeMessage
@ -32,6 +31,7 @@ import {toast} from "react-semantic-toasts-2";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import {SettingsContext} from "../../contexts/contexts";
import {ConfigsTable} from "./Configs";
import {semanticUIColorMap} from "../Vars";
export function ShutdownButton() {
const dockerized = useDockerized();
@ -388,7 +388,7 @@ export function SettingsPage() {
<ButtonGroup>
<Button color={state.nav_color} onClick={e => e.preventDefault()}>Navbar Color</Button>
<Dropdown
id='settings_navbar_color_dropdown'
id='settings_navbar_color_dropdown'
className='button icon'
floating
options={navColorOptions}

77
app/src/hooks/customHooks.js
View file

@ -1,5 +1,6 @@
import React, {useContext, useEffect, useRef, useState} from "react";
import {
createChannel,
fetchDecoded,
fetchDomains,
fetchFilesProgress,
@ -27,6 +28,7 @@ import {
searchZim,
setHotspot,
setThrottle,
updateChannel,
} from "../api";
import {createSearchParams, useLocation, useSearchParams} from "react-router-dom";
import {enumerate, filterToMimetypes, humanFileSize, secondsToFullDuration} from "../components/Common";
@ -35,6 +37,7 @@ import {toast} from "react-semantic-toasts-2";
import {useSearch} from "../components/Search";
import _ from "lodash";
import {TagsSelector} from "../Tags";
import {useForm} from "./useForm";
const calculatePage = (offset, limit) => {
return offset && limit ? Math.round((offset / limit) + 1) : 1;
@ -424,51 +427,55 @@ export const useChannel = (channel_id) => {
name: '',
directory: '',
url: '',
download_frequency: '',
match_regex: '',
tag_name: null,
};
const [ready, setReady] = useState(false);
const [channel, setChannel] = useState(emptyChannel);
const [original, setOriginal] = useState({});
const localGetChannel = async () => {
const fetchChannel = async () => {
if (!channel_id) {
console.debug('Not fetching channel because no channel_id is provided');
return;
}
const c = await getChannel(channel_id);
// Prevent controlled to uncontrolled.
c['url'] = c['url'] || '';
c['download_frequency'] = c['download_frequency'] || '';
c['match_regex'] = c['match_regex'] || '';
return c;
}
const submitChannel = async () => {
const body = {
name: form.formData.name,
directory: form.formData.directory,
url: form.formData.url,
};
if (channel_id) {
try {
const c = await getChannel(channel_id);
// Prevent controlled to uncontrolled.
c['url'] = c['url'] || '';
c['download_frequency'] = c['download_frequency'] || '';
c['match_regex'] = c['match_regex'] || '';
setChannel(c);
setOriginal(c);
setReady(true);
} catch (e) {
setReady(false);
console.error(e);
toast({
type: 'error',
title: 'Unexpected server response',
description: 'Could not get Channel',
time: 5000,
});
}
// Can create a Channel with a Tag.
body.tag_name = form.formData.tag_name;
return await createChannel(body);
} else {
setChannel(emptyChannel);
return await updateChannel(channel_id, body);
}
}
useEffect(() => {
localGetChannel();
const form = useForm({
fetcher: fetchChannel,
emptyFormData: emptyChannel,
clearOnSuccess: false,
submitter: submitChannel
});
React.useEffect(() => {
// Channel may be gotten from Video data, fetch the Channel again.
form.fetcher();
}, [channel_id]);
const changeValue = (name, value) => {
const newChannel = {...channel, [name]: value};
console.debug('useChannel.changeValue', newChannel);
setChannel(newChannel);
}
return {channel, changeValue, original, ready, fetchChannel: localGetChannel};
return {
channel: form.formData,
form,
fetchChannel,
};
}
export const useChannels = () => {

417
app/src/hooks/useForm.js Normal file
View file

@ -0,0 +1,417 @@
import React from "react";
import _ from "lodash";
import {HelpPopup, RequiredAsterisk, Toggle, validURL, validURLs} from "../components/Common";
import {FormInput, TextArea} from "../components/Theme";
async function asyncNoOp() {
}
export function useForm({
fetcher,
submitter,
defaultFormData,
emptyFormData,
onSuccess = asyncNoOp,
onFailure = asyncNoOp,
onDirty = _.noop,
clearOnSuccess = false,
fetchOnSuccess = false,
}) {
const [formData, setFormData] = React.useState(defaultFormData || {});
const [disabled, setDisabled] = React.useState(false);
const [loading, setLoading] = React.useState(false);
const [ready, setReady] = React.useState(false);
const [dirty, setDirty] = React.useState(false);
const [errors, setErrors] = React.useState({});
const [validators, setValidators] = React.useState({});
const [validValues, setValidValues] = React.useState({});
const [requires, setRequires] = React.useState({});
const memoizedFormData = React.useMemo(() => formData, [formData]);
const memoizedErrors = React.useMemo(() => errors, [errors]);
const memoizedValidValues = React.useMemo(() => validValues, [validValues]);
const memoizedRequires = React.useMemo(() => requires, [requires]);
if (clearOnSuccess && fetchOnSuccess) {
console.error('Cannot use both clearOnSuccess and fetchOnSuccess!');
}
if (fetchOnSuccess && !fetcher) {
console.error('Cannot fetchOnSuccess without fetcher!');
}
React.useEffect(() => {
// Form is dirty if it has deviated from the default.
setDirty(!_.isEqual(defaultFormData, memoizedFormData));
const errorValues = Object.values(errors).filter(i => !!i);
if (errorValues.length > 0) {
console.debug('Form invalid because it has errors');
setReady(false);
return
}
const invalidValues = Object.values(memoizedValidValues).filter(i => i !== true);
if (invalidValues.length > 0) {
console.debug('Form invalid because it failed validators');
setReady(false);
return
}
const missingValues = Object.keys(memoizedRequires).filter(i => !_.get(formData, i));
if (missingValues.length > 0) {
console.debug(`Form invalid because it is missing required value: ${missingValues[0]}`);
setReady(false);
return
}
setReady(true);
}, [loading, memoizedFormData, memoizedErrors, memoizedValidValues, memoizedRequires]);
const localFetch = async () => {
if (fetcher) {
const result = await fetcher();
setFormData(result);
}
}
React.useEffect(() => {
localFetch();
}, []);
React.useEffect(() => {
if (dirty) {
onDirty();
}
}, [dirty, onDirty]);
const patchFormData = (newFormData) => {
newFormData = {...formData, ...newFormData};
// console.debug('useForm.patchFormData', newFormData);
setFormData(newFormData);
};
const reset = () => {
setFormData(emptyFormData || defaultFormData);
}
const onSubmit = async () => {
if (!ready) {
console.error('Refusing to submit form because it is not ready');
return;
}
setDisabled(true);
setLoading(true);
setReady(false);
try {
await submitter(formData);
if (fetchOnSuccess && fetcher) {
const result = await fetcher();
setFormData(result);
} else if (clearOnSuccess) {
reset();
}
await onSuccess();
} finally {
setLoading(false);
setDisabled(false);
await onFailure();
}
}
// Using useCallback with dependencies to ensure the debounce function doesn't change on every render
const debouncedValidate = React.useCallback(_.debounce((path, value) => {
const validator = validators[path];
let error = null;
if (validator) {
try {
error = validator(value);
} catch (e) {
console.error(`Failed to validate ${path}`);
}
}
console.debug('useForm.debouncedValidate', path, value, 'error=', error);
setValidValues(prev => ({...prev, [path]: !error}));
setErrors(prev => {
if (error) {
return {...prev, [path]: error};
} else {
const {[path]: _, ...newErrors} = prev;
return newErrors;
}
});
}, 300), [validators]);
const handleInputEvent = (e) => {
if (e) e.preventDefault();
let {type, value} = e.target;
let {path} = e.target.dataset;
path = path || e.target.name;
if (type === 'number' && !isNaN(value)) {
value = parseInt(value);
}
console.debug('handleInputEvent', 'path=', path, 'type=', type, 'value=', value);
// Change value in state first, this part happens immediately
const newFormData = _.set(formData, path.split('.'), value);
patchFormData(newFormData);
// Trigger validation after a delay
debouncedValidate(path, value);
}
const addValidator = (path, validator) => {
if (path && !!validator && !(path in validators)) {
setValidators({...validators, [path]: validator});
}
}
const addRequires = (name) => {
if (name && !(name in requires)) {
setRequires({...requires, [name]: false});
}
}
const getCustomProps = ({name, validator, type = 'text', path, required = false}) => {
// Props for any other elements.
path = path || name;
addValidator(path, validator);
if (required) {
addRequires(path);
}
const value = _.get(formData, path); // Get a path, separated by .'s
if (value === undefined) {
patchFormData(_.set(formData, path, null));
}
// Allow bypass of `handleInputEvent` which only handles an input event.
const setValue = newValue => {
patchFormData(_.set(formData, path, newValue));
debouncedValidate(path, newValue);
}
// Props for <input/>, etc.
const inputProps = {
type,
disabled,
value,
onChange: setValue,
'data-path': path,
}
// Attributes about the input, but should not be passed as properties.
const inputAttrs = {
valid: validValues[path],
path,
setValue,
};
return [inputProps, inputAttrs]
}
const getInputProps = ({name, validator, path, required = false, type = 'text'}) => {
// Props for <input/>
path = path || name;
// Use generic validators if no validator is provided.
if (!validator && type === 'url') {
validator = validURL;
}
const [customProps, inputAttrs] = getCustomProps({name, validator, path, type, required});
// Attributes that should be passed as properties to the input.
const inputProps = {
...customProps,
name: name,
onChange: handleInputEvent,
error: errors[path] || null,
required: required ? null : undefined,
}
return [inputProps, inputAttrs]
}
const getSelectionProps = ({name, validator, path, type, required = false}) => {
// Props for Dropdowns.
path = path || name;
const [customProps, inputAttrs] = getCustomProps({name, validator, type, path, required});
customProps.onChange = (e, {value}) => {
patchFormData(_.set(formData, path, value))
};
return [customProps, inputAttrs]
}
return {
dirty,
disabled,
fetcher: localFetch,
formData,
getCustomProps,
getInputProps,
getSelectionProps,
handleInputEvent,
loading,
onSubmit,
ready,
reset,
}
}
export const commaSeparatedValidator = (value) => {
if (typeof value !== 'string') {
return 'Expected a string';
}
if (value.endsWith(',')) {
return 'Cannot end with comma';
}
if (value.startsWith(',')) {
return 'Cannot start with comma';
}
}
export function InputForm({
form,
required = false,
name,
path,
type = 'text',
validator,
placeholder = '',
label,
helpContent = null,
helpPosition = 'top',
extraInputProps = {},
disabled = false,
}) {
const [inputProps, inputAttrs] = form.getInputProps({name, path, validator, type, required});
inputProps.disabled = disabled || inputProps.disabled;
return <>
<label for={`${name}_input`}>
<b>{label} {required && <RequiredAsterisk/>}</b>
{helpContent &&
<HelpPopup
content={helpContent}
position={helpPosition}
/>
}
</label>
<FormInput
id={`${name}_input`}
placeholder={placeholder}
error={inputProps.error}
>
<input {...extraInputProps} {...inputProps}/>
</FormInput>
</>
}
export function NumberInputForm({
form,
required,
name,
path,
validator,
placeholder = '',
label,
helpContent,
helpPosition,
min = 1,
max = 9999999999,
}) {
// Default to positive number.
validator = validator || ((value) => {
value = _.isNumber(value) ? value : parseInt(value);
if (value < 0) {
return 'Number must be positive'
}
});
return <InputForm
form={form}
required={required}
name={name}
path={path}
type='number'
validator={validator}
placeholder={placeholder}
label={label}
helpContent={helpContent}
helpPosition={helpPosition}
extraInputProps={{min, max}}
/>
}
export function UrlInput({form, required = true, name = 'url', path = 'url', disabled = false}) {
const validator = (i) => {
return validURL(i) ? null : 'Invalid URL';
};
return <InputForm
form={form}
type='url'
label='URL'
required={required}
name={name}
path={path}
validator={validator}
disabled={disabled}
/>
}
export function UrlsTextarea({name = 'urls', required, form}) {
required = required !== undefined;
const validator = (value) => {
if (!validURLs(value)) {
return 'Invalid URLs';
}
}
const [inputProps, inputAttrs] = form.getInputProps({name, validator, required});
const handleDrop = (e) => {
if (e) e.preventDefault();
// Handle user dropping multiple URLs into the textarea. Assume each drop is a URL. Ensure each URL
// is on its own line.
const droppedUrl = e.dataTransfer.getData('text');
let urls = (inputProps.value || '').split('\n');
urls = [...urls, droppedUrl];
urls = urls.filter(i => !!i).join('\n');
// Leave trailing newline for user to type.
inputAttrs.setValue(`${urls}\n`)
};
const handleKeyDown = (event) => {
// Check if Ctrl (or Cmd for Mac) + Enter are pressed
if ((event.ctrlKey || event.metaKey) && event.key === 'Enter') {
event.preventDefault(); // Prevents the addition of a new line in textarea if necessary
form.onSubmit(); // Call your submit handler
}
};
return <FormInput required
error={inputProps.error}
label='URLs'
>
<TextArea
id='urls_textarea'
placeholder={'Enter one URL per line'}
name='urls'
onDrop={handleDrop}
onKeyDown={handleKeyDown}
{...inputProps}
/>
</FormInput>
}
export function ToggleForm({form, name, label, path}) {
const [inputProps, inputAttrs] = form.getCustomProps({name, path})
return <FormInput
label={label}>
<Toggle
disabled={form.disabled}
name={name}
checked={inputProps.value}
/>
</FormInput>
}

17
app/src/setupTests.js
View file

@ -1,17 +0,0 @@
// jest-dom adds custom jest matchers for asserting on DOM nodes.
// allows you to do things like:
// expect(element).toHaveTextContent(/react/i)
// learn more: https://github.com/testing-library/jest-dom
import '@testing-library/jest-dom';
window.matchMedia = window.matchMedia || function () {
return {
matches: false,
addListener: function () {
},
addEventListener: function () {
},
removeListener: function () {
}
};
};

2
etc/raspberrypios/wrolpi-api.service
View file

@ -5,7 +5,7 @@ PartOf=wrolpi.target
[Service]
Environment=PYTHONUNBUFFERED=1
ExecStart=/opt/wrolpi/venv/bin/sanic main:api_app --host 0.0.0.0 --port 8081 -w 3
ExecStart=/opt/wrolpi/venv/bin/sanic main:api_app --host 0.0.0.0 --port 8081 -w 5
ExecReload=/bin/kill -HUP $MAINPID
Group=wrolpi
User=wrolpi

4
main.py
View file

@ -58,12 +58,12 @@ This is the interactive WROLPi shell. Use this to interact with the WROLPi API
Example (get the duration of every video file):
from modules.videos.models import Video
from modules.videos.common import get_video_duration
from modules.videos.common import extract_video_duration
videos = session.query(Video).all()
videos = list(videos)
for video in videos:
get_video_duration(video.file_group.primary_path)
extract_video_duration(video.file_group.primary_path)
Check local variables:
locals().keys()

6
modules/archive/__init__.py
View file

@ -54,11 +54,13 @@ class ArchiveDownloader(Downloader, ABC):
with get_db_session() as session:
archive = Archive.find_by_id(archive_id, session)
if download.settings and (tag_names := download.settings.get('tag_names')):
need_commit = False
if tag_names := download.tag_names:
for name in tag_names:
archive.add_tag(name, session)
need_commit = True
if session := Session.object_session(archive):
if need_commit:
session.commit()
logger.info(f'Successfully downloaded Archive {download.url} {archive}')

26
modules/videos/channel/test/test_api.py
View file

@ -18,6 +18,7 @@ from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.downloader import download_manager, Download, DownloadResult
from wrolpi.files.models import Directory
from wrolpi.switches import await_switches
from wrolpi.test.common import assert_dict_contains
@ -218,8 +219,7 @@ async def test_channel_download_requires_refresh(
downloaded_urls.append(download.url)
return DownloadResult(success=True)
with mock.patch('modules.videos.downloader.VideoDownloader.do_download', do_download), \
mock.patch('wrolpi.files.lib.apply_indexers'): # TODO why does apply_indexers break this test?
with mock.patch('modules.videos.downloader.VideoDownloader.do_download', do_download):
entries = [
dict(id=vid.source_id, view_count=0, webpage_url='https://example.com/1'), # Already downloaded.
dict(id='not downloaded', view_count=0, webpage_url='https://example.com/2'),
@ -228,8 +228,7 @@ async def test_channel_download_requires_refresh(
channel_id='the id', id='the id')
await video_download_manager.do_downloads()
await video_download_manager.wait_for_all_downloads()
# Give time for background tasks to finish. :(
await asyncio.sleep(1)
await await_switches()
# Channel was refreshed before downloading videos.
assert_refreshed(True)
@ -465,13 +464,12 @@ async def test_tag_channel(async_client, test_session, test_directory, channel_f
v1, v2 = video_factory(title='video1', channel_id=channel.id), video_factory(title='video2', channel_id=channel.id)
# Create recurring download which uses the Channel's directory.
download = download_manager.recurring_download('https://example.com/1', 60, test_downloader.name,
settings={
'destination': str(test_directory / 'videos/Channel Name')})
destination=str(test_directory / 'videos/Channel Name'))
test_session.commit()
save_channels_config()
assert get_channels_config().channels[0]['directory'] == str(test_directory / 'videos/Channel Name')
# Channel download downloads into the Channel's directory.
assert channel.downloads[0].settings['destination'] == str(test_directory / 'videos/Channel Name')
assert channel.downloads[0].destination == test_directory / 'videos/Channel Name'
# Make extra file in the Channel's directory, it should be moved.
(channel_directory / 'extra file.txt').write_text('extra file contents')
# Channel directory is in the Videos directory.
@ -498,10 +496,8 @@ async def test_tag_channel(async_client, test_session, test_directory, channel_f
assert not channel_directory.exists(), 'Old Channel directory should have been deleted.'
# Channel download goes into the Channel's directory.
d1, d2 = test_session.query(Download).all()
assert d1.settings['destination'] == str(test_directory / 'videos/Tag Name/Channel Name'), \
f'{d1} was not moved'
assert d2.settings['destination'] == str(test_directory / 'videos/Tag Name/Channel Name'), \
f'{d2} was not moved'
assert d1.destination == test_directory / 'videos/Tag Name/Channel Name', f'{d1} was not moved'
assert d2.destination == test_directory / 'videos/Tag Name/Channel Name', f'{d2} was not moved'
# Directory record was replaced.
assert [i.path for i in test_session.query(Directory)] == [channel.directory, ]
@ -529,8 +525,8 @@ async def test_tag_channel(async_client, test_session, test_directory, channel_f
assert not (test_directory / 'videos/Tag Name/Channel Name').exists()
# Downloads are moved back.
d1, d2 = test_session.query(Download).all()
assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
assert d1.destination == test_directory / 'videos/Channel Name', f'{d1} was not moved'
assert d2.destination == test_directory / 'videos/Channel Name', f'{d2} was not moved'
assert [i.path for i in test_session.query(Directory)] == [channel.directory, ]
# Channel can be Tagged, without moving directories.
@ -544,8 +540,8 @@ async def test_tag_channel(async_client, test_session, test_directory, channel_f
assert str(get_relative_to_media_directory(v2.video_path)) == 'videos/Channel Name/video2.mp4'
# Downloads were not changed.
d1, d2 = test_session.query(Download).all()
assert d1.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d1} was not moved'
assert d2.settings['destination'] == str(test_directory / 'videos/Channel Name'), f'{d2} was not moved'
assert d1.destination == test_directory / 'videos/Channel Name', f'{d1} was not moved'
assert d2.destination == test_directory / 'videos/Channel Name', f'{d2} was not moved'
assert [i.path for i in test_session.query(Directory)] == [channel.directory, ]

2
modules/videos/channel/test/test_models.py
View file

@ -88,7 +88,7 @@ async def test_channel_download_relationships(test_session, download_channel):
assert len(channel.downloads) == 1
assert channel.downloads[0] == download and channel.downloads[0].url == 'https://example.com/channel1'
assert download.frequency == DownloadFrequency.weekly
assert download.settings['destination'] == str(channel.directory)
assert download.destination == channel.directory
# Deleting Download deletes the Download, but not the Channel.
download.delete()

4
modules/videos/common.py
View file

@ -147,7 +147,7 @@ def ffmpeg_video_complete(video_path: Path, seconds: int = None) -> bool:
path = Path(fh.name)
path.unlink()
try:
seconds = seconds or get_video_duration(video_path) - 5
seconds = seconds or extract_video_duration(video_path) - 5
ffmpeg_poster(video_path, path, seconds)
return True
except subprocess.CalledProcessError:
@ -243,7 +243,7 @@ async def ffprobe_json(video_path: Union[Path, str]) -> dict:
return content
def get_video_duration(video_path: Path) -> Optional[int]:
def extract_video_duration(video_path: Path) -> Optional[int]:
"""Get the duration of a video in seconds. Do this using ffprobe."""
if not isinstance(video_path, Path):
video_path = Path(video_path)

233
modules/videos/downloader.py
View file

@ -1,19 +1,23 @@
#! /usr/bin/env python3
import json
import logging
import os.path
import pathlib
import traceback
from abc import ABC
from datetime import timedelta
from typing import Tuple, List, Dict, Optional
import cachetools
import yt_dlp.utils
from cachetools.keys import hashkey
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL, DownloadError
from yt_dlp.extractor import YoutubeTabIE # noqa
from wrolpi.cmd import YT_DLP_BIN
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \
format_json_file, trim_file_name
trim_file_name, cached_multiprocessing_result
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.db import optional_session
@ -23,10 +27,10 @@ from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
from wrolpi.files.models import FileGroup
from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR
from .channel.lib import create_channel, get_channel
from .common import get_no_channel_directory, get_videos_directory, update_view_counts_and_censored, \
from .common import get_no_channel_directory, update_view_counts_and_censored, \
ffmpeg_video_complete
from .errors import UnknownChannel
from .lib import get_downloader_config, YDL, ydl_logger
from .lib import get_downloader_config, YDL, ydl_logger, format_videos_destination
from .models import Video, Channel
from .normalize_video_url import normalize_video_url
from .schema import ChannelPostRequest
@ -34,27 +38,41 @@ from .video.lib import download_video_info_json
logger = logger.getChild(__name__)
# Channels are handled differently than a single video.
ChannelIEs = {
YoutubeTabIE,
VIDEO_RESOLUTION_MAP = {
'360p': ['134+140', 'mp4-360p', 'res:360'],
'480p': ['135+140', '135+139', 'mp4-480p', 'res:480'],
'720p': [
'298+140', # 720@60 avc1 + 135k audio
'136+140', '22', 'res:720'],
'1080p': [
'299+140', # 1080@60 avc1 + 135k audio
'137+140', '614+140', 'res:1080'],
'1440p': [
'639+140', # vp9 + 135k audio
'620+140', 'res:1440'],
'2160p': [
'642+140', # vp9 + 135k audio
'625+140', 'res:2160'],
'maximum': ['bestvideo*+bestaudio/best'],
}
PREFERRED_VIDEO_EXTENSION = 'mp4'
PREFERRED_VIDEO_FORMAT = ','.join([
'res:720', # Use the best 720p available first
'136+140', # 130=720p video-only, 140=medium quality audio only
'22', # 720p video with audio
'135+140', # 135=480p video-only, 140=medium quality audio only
'135+139', # 135=480p video-only, 139=low quality audio only
'mp4-480p',
'bestvideo*+bestaudio/best', # Download the highest resolution as a last resort (can be quite large).
])
PREFERRED_CAPTION_FORMAT = 'vtt'
PREFERRED_POSTER_FORMAT = 'jpg'
# VTT is the only format that a browser can use to display captions on a <video/>.
DEFAULT_CAPTION_FORMAT = 'vtt'
DEFAULT_POSTER_FORMAT = 'jpg'
DEFAULT_CHANNEL_DOWNLOAD_ORDER = 'newest'
def extract_info(url: str, ydl: YoutubeDL = YDL, process=False) -> dict:
"""Get info about a video. Separated for testing."""
def extract_info_key(url: str, ydl: YoutubeDL = YDL, process: bool = False):
return hashkey(url, process=process)
extract_info_cache = cachetools.TTLCache(maxsize=1_000, ttl=timedelta(minutes=5).total_seconds())
# Cache results for each URL for a few minutes.
@cachetools.cached(cache=extract_info_cache, key=extract_info_key)
def extract_info(url: str, ydl: YoutubeDL = YDL, process: bool = False) -> dict:
"""Get info about a video, channel, or playlist. Separated for testing."""
if PYTEST:
raise RuntimeError(f'Refusing to download {url} during testing! {ydl}')
@ -69,6 +87,25 @@ def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
return f'{dir_name}/{file_name}'
@cached_multiprocessing_result
async def fetch_video_duration(url: str) -> int:
"""Get video duration in seconds. Attempts to get the duration from a Video that has already been downloaded,
otherwise, fetches the duration using yt-dlp."""
with get_db_session() as session:
# Join with Video to get duration; a URL may have other files (such as an Archive).
video = Video.get_by_url(url, session=session)
if video and (duration := video.file_group.length):
logger.debug(f'Using already-known duration {duration} for {url} from FileGroup')
return duration
logger.debug(f'Fetching video duration from {url}')
# Duration can be extracted without processing.
info = extract_info(url, process=False)
duration = info['duration']
logger.info(f'Fetched video duration of {duration} from {url}')
return duration
class ChannelDownloader(Downloader, ABC):
"""Handles downloading of videos in a Channel or Playlist."""
name = 'video_channel'
@ -87,7 +124,7 @@ class ChannelDownloader(Downloader, ABC):
"""Update a Channel's catalog, then schedule downloads of every missing video."""
info = extract_info(download.url, process=False)
# Resolve the "entries" generator.
info: Dict = resolve_generators(info)
info: dict = resolve_generators(info)
entries = info.get('entries')
if entries and isinstance(entries[0], Dict) and entries[0].get('entries'):
# Entries contains more channels/playlist, tell the Maintainer to specify the URL.
@ -95,6 +132,9 @@ class ChannelDownloader(Downloader, ABC):
f' Try requesting the videos only:'
f' https://example.com -> https://example.com/videos')
download_settings = download.settings or dict()
channel_tag_name = i[0] if (i := download_settings.get('channel_tag_name')) else None
download.sub_downloader = video_downloader.name
download.info_json = info
if session := Session.object_session(download):
@ -105,7 +145,7 @@ class ChannelDownloader(Downloader, ABC):
if not name:
raise ValueError(f'Could not find name')
channel_source_id = info.get('channel_id') or info.get('id')
channel = get_or_create_channel(channel_source_id, download.url, name)
channel = get_or_create_channel(channel_source_id, download.url, name, channel_tag_name)
channel.dict() # get all attributes while we have the session.
location = f'/videos/channel/{channel.id}/video' if channel and channel.id else None
@ -114,14 +154,14 @@ class ChannelDownloader(Downloader, ABC):
settings = dict()
if channel:
settings.update(dict(channel_id=channel.id, channel_url=download.url))
# Downloads will inherit the destination, if defined.
destination = download.settings.get('destination') if download.settings else None
if destination:
settings['destination'] = destination
# A Download may have applied tags, send any to the VideoDownloader.
tag_names = download.settings.get('tag_names') if download.settings else None
if tag_names:
settings['tag_names'] = tag_names
if download.destination:
settings['destination'] = str(download.destination) # Need str for JSON conversion
if download.tag_names:
settings['tag_names'] = download.tag_names
if video_resolutions := download_settings.get('video_resolutions'):
settings['video_resolutions'] = video_resolutions
if video_format := download_settings.get('video_format'):
settings['video_format'] = video_format
is_a_playlist = self.is_a_playlist(info)
try:
@ -173,28 +213,49 @@ class ChannelDownloader(Downloader, ABC):
downloads = download.info_json['entries']
total_downloads = len(downloads)
title_include = download.settings.get('title_include') if download.settings else None
if title_include:
# Only download Videos that have title match words in title.
title_include = [i.strip() for i in title_include.split(',')]
settings = download.settings or dict()
title_exclude = settings.get('title_exclude', '')
title_exclude = [i.lower() for i in title_exclude.split(',') if i]
title_include = settings.get('title_include', '')
title_include = [i.lower() for i in title_include.split(',') if i]
if title_exclude or title_include:
new_downloads = []
for i in downloads:
if (title := i.get('title', '')) and any(j for j in title_include if j in title):
logger.debug(f'Video with title {title} matches {title_include}')
new_downloads.append(i)
title = i.get('title', '').lower()
if title and title_exclude and any(i in title for i in title_exclude):
logger.debug(f'Video with title {str(repr(title))} matches {title_exclude=}')
continue
if title and title_include and not any(i in title for i in title_include):
logger.debug(f'Video with title {str(repr(title))} matches {title_include=}')
continue
new_downloads.append(i)
downloads = new_downloads
title_exclude = download.settings.get('title_exclude') if download.settings else None
if title_exclude:
# Do not download any videos that contain the exclude words.
title_exclude = [i.strip() for i in title_exclude.split(',')]
new_downloads = list()
for i in downloads:
if (title := i.get('title', '')) and any(j for j in title_exclude if j in title):
logger.debug(f'Video with title {title} matches EXCLUDE {title_exclude}')
else:
new_downloads.append(i)
downloads = new_downloads
# Filter videos by their length.
try:
if minimum_duration := settings.get('minimum_duration'):
downloads = [i for i in downloads if int(i['duration']) >= minimum_duration]
if maximum_duration := settings.get('maximum_duration'):
downloads = [i for i in downloads if int(i['duration']) <= maximum_duration]
except KeyError as e:
raise RuntimeError('Unable to filter videos because there is no duration') from e
# Sort videos now that we have reduced the number necessary to sort, but before we limit count.
sort_key = settings.get('download_order', DEFAULT_CHANNEL_DOWNLOAD_ORDER)
# Default is videos are sorted by newest first.
if sort_key == 'oldest':
logger.debug(f'Downloading oldest videos from {download.url}')
downloads = downloads.reverse()
elif sort_key == 'views':
# Download videos with most views.
logger.debug(f'Downloading most viewed videos from {download.url}')
downloads = sorted(downloads, key=lambda i: i['view_count'], reverse=True)
# Limit the videos that will be downloads (this allows the user to download the "top 100" videos of a Channel)
if video_count_limit := settings.get('video_count_limit'):
logger.debug(f'Limiting video count to {video_count_limit}: {download.url}')
video_count_limit = int(video_count_limit)
downloads = downloads[:video_count_limit]
filtered_entries = len(downloads)
if filtered_entries != total_downloads:
@ -240,6 +301,11 @@ class VideoDownloader(Downloader, ABC):
download_id = download.id
url = normalize_video_url(download.url)
# Copy settings into Download (they may be from the ChannelDownloader)
settings = download.settings or dict()
download.destination = download.destination or settings.get('destination')
tag_names = download.tag_names = download.tag_names or settings.get('tag_names')
# Video may have been downloaded previously, get its location for error reporting.
with get_db_session() as session:
video_ = Video.get_by_url(url, session)
@ -265,8 +331,8 @@ class VideoDownloader(Downloader, ABC):
if destination:
# Download to the directory specified in the settings.
out_dir = pathlib.Path(settings['destination'])
logger.debug(f'Downloading {url} to destination from settings')
out_dir = destination
logger.debug(f'Downloading {url} to destination {destination} from settings')
elif channel:
out_dir = channel_directory
logger.debug(f'Downloading {url} to channel directory: {channel_directory}')
@ -275,11 +341,24 @@ class VideoDownloader(Downloader, ABC):
out_dir = get_no_channel_directory()
logger.debug(f'Downloading {url} to default directory')
# Make output directory. (Maybe string from settings)
out_dir = out_dir if isinstance(out_dir, pathlib.Path) else pathlib.Path(out_dir)
out_dir.mkdir(exist_ok=True, parents=True)
if tag_names and logger.isEnabledFor(logging.DEBUG):
logger.debug(f'Downloading {url} with {tag_names=}')
config = get_downloader_config()
logs = None # noqa
try:
video_path, entry = self.prepare_filename(url, out_dir)
# Use user-provided/channel-provided first, or fallback to defaults from config.
video_resolutions = settings.get('video_resolutions') or config.video_resolutions
# yt-dlp expects a string like so: 299+140,298+140,bestvideo*+bestaudio/best
video_resolutions = ','.join(j for i in video_resolutions for j in VIDEO_RESOLUTION_MAP[i])
video_format = settings.get('video_format') or config.merge_output_format
video_path, entry = self.prepare_filename(url, out_dir, video_resolutions, video_format)
if settings.get('download_metadata_only'):
# User has requested refresh of video metadata, skip the rest of the video downloader.
@ -289,18 +368,17 @@ class VideoDownloader(Downloader, ABC):
cmd = (
str(YT_DLP_BIN),
'-c', # Continue downloads
'-f', PREFERRED_VIDEO_FORMAT,
'-f', video_resolutions,
'--match-filter', '!is_live', # Do not attempt to download Live videos.
'--write-subs',
'--write-auto-subs',
'--sub-format', PREFERRED_CAPTION_FORMAT,
'--convert-subs', PREFERRED_CAPTION_FORMAT,
'--convert-thumbnails', PREFERRED_POSTER_FORMAT,
'--sub-format', DEFAULT_CAPTION_FORMAT,
'--convert-subs', DEFAULT_CAPTION_FORMAT,
'--convert-thumbnails', DEFAULT_POSTER_FORMAT,
'--write-thumbnail',
'--write-info-json',
'--merge-output-format', PREFERRED_VIDEO_EXTENSION,
'--remux-video', PREFERRED_VIDEO_EXTENSION,
'-o', video_path,
'--merge-output-format', video_format,
# '--remux-video', video_format,
'--no-cache-dir',
'--compat-options', 'no-live-chat',
# Get top 20 comments, 10 replies per parent.
@ -308,6 +386,7 @@ class VideoDownloader(Downloader, ABC):
'--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top',
# Use experimental feature to merge files.
'--ppa', 'Merger+ffmpeg_o1:-strict -2',
'-o', video_path,
url,
)
return_code, logs, _ = await self.process_runner(download_id, url, cmd, out_dir)
@ -323,7 +402,7 @@ class VideoDownloader(Downloader, ABC):
location=location,
)
preferred_path = video_path.with_suffix(f'.{PREFERRED_VIDEO_EXTENSION}')
preferred_path = video_path.with_suffix(f'.{video_format}')
if not video_path.is_file() and preferred_path.is_file():
# Prepared filename does not exist, but video with preferred video extension does, it was probably
# remuxed by yt-dlp.
@ -364,7 +443,7 @@ class VideoDownloader(Downloader, ABC):
with get_db_session(commit=True) as session:
# Second session is started because SQLAlchemy will forget what we have done.
video = Video.get_by_id(video_id, session)
if download.settings and (tag_names := download.settings.get('tag_names')):
if tag_names:
existing_names = video.file_group.tag_names
for name in tag_names:
if name not in existing_names:
@ -428,6 +507,9 @@ class VideoDownloader(Downloader, ABC):
@staticmethod
async def _get_channel(download: Download) \
-> Tuple[Optional[Channel], Optional[pathlib.Path], Optional[int], Optional[pathlib.Path], dict]:
settings = download.settings or dict()
channel_tag_name = i[0] if (i := settings.get('channel_tag_name')) else None
found_channel = None
# Look for Channel using channel data first, fallback to uploader data.
channel_name = download.info_json.get('channel') or download.info_json.get('uploader')
@ -437,15 +519,15 @@ class VideoDownloader(Downloader, ABC):
if channel_name or channel_source_id or channel_url:
# Try to find the channel via info_json from yt-dlp.
try:
channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name)
channel = get_or_create_channel(source_id=channel_source_id, url=channel_url, name=channel_name,
tag_name=channel_tag_name)
if channel:
found_channel = 'yt_dlp'
logger.debug(f'Found a channel with source_id {channel=}')
except UnknownChannel:
# Can't find a channel, use the no channel directory.
pass
settings = download.settings or dict()
destination = settings.get('destination')
destination = download.destination
if not channel and destination:
# Destination may find Channel if not already found.
try:
@ -455,7 +537,7 @@ class VideoDownloader(Downloader, ABC):
except UnknownChannel:
# Destination must not be a channel.
pass
channel_id = settings.get('channel_id')
channel_id = download.channel_id
channel_url = settings.get('channel_url')
if not channel and (channel_id or channel_url):
# Could not find Channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
@ -469,6 +551,7 @@ class VideoDownloader(Downloader, ABC):
raise
if channel:
found_channel = 'download_settings'
channel_id = channel.id if channel else None
channel_directory = channel.directory if channel else None
if found_channel == 'yt_dlp':
@ -483,17 +566,20 @@ class VideoDownloader(Downloader, ABC):
return channel, channel_directory, channel_id, destination, settings
@staticmethod
def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]:
def prepare_filename(url: str, out_dir: pathlib.Path, video_resolutions: str, video_format: str) \
-> Tuple[pathlib.Path, dict]:
"""Get the full path of a video file from its URL using yt-dlp."""
if not out_dir.is_dir():
raise ValueError(f'Output directory does not exist! {out_dir=}')
# YoutubeDL expects specific options, add onto the default options
options = get_downloader_config().dict()
options['outtmpl'] = f'{out_dir}/{options["file_name_format"]}'
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
options['remuxvideo'] = PREFERRED_VIDEO_EXTENSION
options['format'] = PREFERRED_VIDEO_FORMAT
config = get_downloader_config()
options = config.yt_dlp_options
# yt-dlp expects the absolute path.
options['outtmpl'] = f'{out_dir}/{config.file_name_format}'
options['merge_output_format'] = video_format
# options['remuxvideo'] = video_format
options['format'] = video_resolutions
options['cachdir'] = YTDLP_CACHE_DIR
# Create a new YoutubeDL for the output directory.
@ -523,7 +609,7 @@ class VideoDownloader(Downloader, ABC):
# Trim long filename, add video suffix, add back into parent directory.
filename = escape_file_name(filename)
final_filename = parent / f'{filename}.{PREFERRED_VIDEO_EXTENSION}'
final_filename = parent / f'{filename}.{video_format}'
final_filename = trim_file_name(final_filename)
logger.debug(f'Video file name was too long. Trimmed to: {final_filename.name}')
@ -586,7 +672,7 @@ channel_downloader = ChannelDownloader()
video_downloader = VideoDownloader()
def get_or_create_channel(source_id: str = None, url: str = None, name: str = None) -> Channel:
def get_or_create_channel(source_id: str = None, url: str = None, name: str = None, tag_name: str = None) -> Channel:
"""
Attempt to find a Channel using the provided params. The params are in order of reliability.
@ -604,7 +690,7 @@ def get_or_create_channel(source_id: str = None, url: str = None, name: str = No
raise UnknownChannel(f'Cannot create channel without a name')
# Channel does not exist. Create one in the video directory.
channel_directory = get_videos_directory() / escape_file_name(name)
channel_directory = format_videos_destination(name, tag_name, url)
if not channel_directory.is_dir():
channel_directory.mkdir(parents=True)
data = ChannelPostRequest(
@ -612,6 +698,7 @@ def get_or_create_channel(source_id: str = None, url: str = None, name: str = No
name=name,
url=url,
directory=str(channel_directory.relative_to(get_media_directory())),
tag_name=tag_name,
)
channel = create_channel(data=data, return_dict=False)
# Create the directory now that the channel is approved.

125
modules/videos/lib.py
View file

@ -3,9 +3,9 @@ import dataclasses
import html
import pathlib
import re
from dataclasses import dataclass
from dataclasses import dataclass, field
from datetime import datetime
from typing import Generator, Type
from typing import Generator, Type, List
from typing import Tuple, Optional
import pytz
@ -15,7 +15,7 @@ from yt_dlp import YoutubeDL
from modules.videos.models import Video
from wrolpi import dates, flags
from wrolpi.captions import extract_captions
from wrolpi.common import ConfigFile, extract_domain, \
from wrolpi.common import ConfigFile, extract_domain, logger, \
escape_file_name, get_media_directory, background_task, Base, get_wrolpi_config
from wrolpi.dates import Seconds, from_timestamp
from wrolpi.db import get_db_curs, get_db_session
@ -26,8 +26,8 @@ from wrolpi.files.lib import split_path_stem_and_suffix
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST
from .common import is_valid_poster, convert_image, \
generate_video_poster, logger, ConfigError, \
get_video_duration
generate_video_poster, ConfigError, \
extract_video_duration
from .errors import UnknownChannel
from .models import Channel
@ -151,7 +151,7 @@ def validate_video(video: Video, channel_generate_poster: bool):
video.file_group.length = float(video_streams[0]['duration'])
else:
# Slowest method.
video.file_group.length = get_video_duration(video_path)
video.file_group.length = extract_video_duration(video_path)
if video_path and not video.caption_paths and video.file_group.d_text and not EXTRACT_SUBTITLES:
# Caption file was deleted, clear out old captions.
@ -276,7 +276,7 @@ class ChannelsConfig(ConfigFile):
session.add(channel)
channel.flush()
# TODO refresh the files in the channel.
logger.warning(f'Creating new Channel from config {channel}')
logger.warning(f'Creating new Channel from config: {directory}')
# Copy existing channel data, update all values from the config. This is necessary to clear out
# values not in the config.
@ -344,9 +344,8 @@ def set_test_channels_config():
@dataclass
class VideoDownloaderConfigValidator:
class VideoDownloaderConfigYtDlpOptionsValidator:
continue_dl: bool
dateafter: str
file_name_format: str
nooverwrites: bool
quiet: bool
@ -355,105 +354,90 @@ class VideoDownloaderConfigValidator:
writesubtitles: bool
writethumbnail: bool
youtube_include_dash_manifest: bool
@dataclass
class VideoDownloaderConfigValidator:
video_resolutions: List[str] = field(default_factory=lambda: ['1080p', '720p', '480p', '360p', 'maximum'])
version: int = None
yt_dlp_options: VideoDownloaderConfigYtDlpOptionsValidator = field(default_factory=dict)
class VideoDownloaderConfig(ConfigFile):
file_name = 'videos_downloader.yaml'
default_config = dict(
continue_dl=True,
dateafter='19900101',
file_name_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s',
nooverwrites=True,
quiet=False,
video_resolutions=['1080p', '720p', '480p', '360p', 'maximum'],
version=0,
writeautomaticsub=True,
writeinfojson=True,
writesubtitles=True,
writethumbnail=True,
youtube_include_dash_manifest=False,
yt_dlp_options=dict(
continue_dl=True,
file_name_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s',
merge_output_format='mp4',
nooverwrites=True,
quiet=False,
writeautomaticsub=True,
writeinfojson=True,
writesubtitles=True,
writethumbnail=True,
youtube_include_dash_manifest=False,
),
)
validator = VideoDownloaderConfigValidator
@property
def continue_dl(self) -> bool:
return self._config['continue_dl']
@continue_dl.setter
def continue_dl(self, value: bool):
self.update({'continue_dl': value})
return self._config['yt_dlp_options']['continue_dl']
@property
def dateafter(self) -> str:
return self._config['dateafter']
def video_resolutions(self) -> List[str]:
return self._config['video_resolutions']
@dateafter.setter
def dateafter(self, value: str):
self.update({'dateafter': value})
@video_resolutions.setter
def video_resolutions(self, value: List[str]):
self.update({'video_resolutions': value})
@property
def file_name_format(self) -> str:
return self._config['file_name_format']
@file_name_format.setter
def file_name_format(self, value: str):
self.update({'file_name_format': value})
return self._config['yt_dlp_options']['file_name_format']
@property
def nooverwrites(self) -> bool:
return self._config['nooverwrites']
return self._config['yt_dlp_options']['nooverwrites']
@nooverwrites.setter
def nooverwrites(self, value: bool):
self.update({'nooverwrites': value})
@property
def merge_output_format(self) -> bool:
return self._config['yt_dlp_options']['merge_output_format']
@property
def quiet(self) -> bool:
return self._config['quiet']
@quiet.setter
def quiet(self, value: bool):
self.update({'quiet': value})
return self._config['yt_dlp_options']['quiet']
@property
def writeautomaticsub(self) -> bool:
return self._config['writeautomaticsub']
@writeautomaticsub.setter
def writeautomaticsub(self, value: bool):
self.update({'writeautomaticsub': value})
return self._config['yt_dlp_options']['writeautomaticsub']
@property
def writeinfojson(self) -> bool:
return self._config['writeinfojson']
@writeinfojson.setter
def writeinfojson(self, value: bool):
self.update({'writeinfojson': value})
return self._config['yt_dlp_options']['writeinfojson']
@property
def writesubtitles(self) -> bool:
return self._config['writesubtitles']
@writesubtitles.setter
def writesubtitles(self, value: bool):
self.update({'writesubtitles': value})
return self._config['yt_dlp_options']['writesubtitles']
@property
def writethumbnail(self) -> bool:
return self._config['writethumbnail']
@writethumbnail.setter
def writethumbnail(self, value: bool):
self.update({'writethumbnail': value})
return self._config['yt_dlp_options']['writethumbnail']
@property
def youtube_include_dash_manifest(self) -> bool:
return self._config['youtube_include_dash_manifest']
return self._config['yt_dlp_options']['youtube_include_dash_manifest']
@youtube_include_dash_manifest.setter
def youtube_include_dash_manifest(self, value: bool):
self.update({'youtube_include_dash_manifest': value})
@property
def yt_dlp_options(self) -> dict:
return self._config['yt_dlp_options']
@yt_dlp_options.setter
def yt_dlp_options(self, value: dict):
self.update({'yt_dlp_options': value})
VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig()
@ -528,11 +512,11 @@ def import_channels_config():
channel_import_logger.info('Importing channels config completed')
def link_channel_and_downloads(session: Session, channel_: Type[Base] = Channel):
def link_channel_and_downloads(session: Session, channel_: Type[Base] = Channel, download_: Type[Base] = Download):
"""Create any missing Downloads for any Channel.url/Channel.directory that has a Download. Associate any Download
related to a Channel."""
# Only Downloads with a frequency can be a Channel Download.
downloads = list(session.query(Download).filter(Download.frequency.isnot(None)).all())
downloads = list(session.query(download_).filter(download_.frequency.isnot(None)).all())
# Download.url is unique and cannot be null.
downloads_by_url = {i.url: i for i in downloads}
# Many Downloads may share the same destination.
@ -738,7 +722,8 @@ def format_videos_destination(channel_name: str = None, channel_tag: str = None,
try:
videos_destination = videos_destination % variables
except KeyError as e:
raise FileNotFoundError(f'Cannot download to the defined "videos_destination"') from e
msg = f'Cannot download to the "videos_destination" from Settings: {videos_destination}'
raise FileNotFoundError(msg) from e
videos_destination = get_media_directory() / videos_destination.lstrip('/')

21
modules/videos/models.py
View file

@ -552,6 +552,10 @@ class Channel(ModelHelper, Base):
def tag_name(self) -> str | None:
return self.tag.name if self.tag else None
@property
def location(self) -> str:
return f'/videos/channel/{self.id}/video'
def delete_with_videos(self):
"""Delete all Video records (but not video files) related to this Channel. Then delete the Channel."""
session = Session.object_session(self)
@ -658,6 +662,7 @@ class Channel(ModelHelper, Base):
def dict(self, with_statistics: bool = False, with_downloads: bool = True) -> dict:
d = super(Channel, self).dict()
d['tag_name'] = self.tag_name
d['rss_url'] = self.get_rss_url()
d['directory'] = \
self.directory.relative_to(get_media_directory()) if self.directory else None
if with_statistics:
@ -767,7 +772,7 @@ class Channel(ModelHelper, Base):
if not download:
download = download_manager.recurring_download(url, frequency, ChannelDownloader.name, session=session,
sub_downloader_name=VideoDownloader.name,
settings=dict(destination=str(self.directory)),
destination=self.directory,
reset_attempts=reset_attempts,
)
if reset_attempts:
@ -778,7 +783,9 @@ class Channel(ModelHelper, Base):
def get_rss_url(self) -> str | None:
"""Return the RSS Feed URL for this Channel, if any is possible."""
if self.url and self.source_id and 'youtube.com' in self.url:
yt_url = 'youtube.com' in self.url if self.url else False
yt_source_id = self.source_id.startswith('UC') if self.source_id else False
if yt_url and yt_source_id:
return f'https://www.youtube.com/feeds/videos.xml?channel_id={self.source_id}'
@staticmethod
@ -806,11 +813,7 @@ class Channel(ModelHelper, Base):
downloads.extend(Download.get_all_by_destination(from_directory))
downloads = unique_by_predicate(downloads, lambda i: i.id)
for download in downloads:
# Get new copy of `settings` to avoid bug where it wouldn't update.
settings = download.settings.copy() if download.settings else dict()
if settings.get('destination'):
logger.debug(f'Updating destination of {download}')
download.settings = {**settings, 'destination': str(to_directory)}
download.destination = to_directory
session.flush(downloads)
# Only one tag can be moved at a time.
@ -858,3 +861,7 @@ class Channel(ModelHelper, Base):
if old_directory.exists() and not next(iter(old_directory.iterdir()), None):
# Old directory is empty, delete it.
old_directory.rmdir()
@staticmethod
def get_by_source_id(session: Session, source_id: str) -> Optional['Channel']:
return session.query(Channel).filter_by(source_id=source_id).one_or_none()

8
modules/videos/test/test_api.py
View file

@ -148,16 +148,16 @@ def test_channels_with_videos(test_session, test_client, test_directory, channel
def test_api_download(test_session, test_client, test_directory):
"""A video can be downloaded."""
content = dict(urls=['https://example.com/video1', ], downloader='video',
settings=dict(excluded_urls='example.com', destination='dest'))
content = dict(urls=['https://example.com/video1', ], downloader='video', destination='dest',
settings=dict(excluded_urls='example.com'))
request, response = test_client.post('/api/download', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
download = test_session.query(Download).one()
assert download.url == 'https://example.com/video1'
assert download.downloader == 'video'
assert download.settings['excluded_urls'] == ['example.com']
assert download.settings['destination'] == str(test_directory / 'dest')
assert download.settings['excluded_urls'] == 'example.com'
assert download.destination == test_directory / 'dest'
assert not download.settings.get('tag_names')

18
modules/videos/test/test_common.py
View file

@ -14,7 +14,7 @@ from wrolpi.files import lib as files_lib
from wrolpi.switches import await_switches
from wrolpi.vars import PROJECT_DIR
from .. import common
from ..common import convert_image, update_view_counts_and_censored, get_video_duration, generate_video_poster, \
from ..common import convert_image, update_view_counts_and_censored, extract_video_duration, generate_video_poster, \
is_valid_poster
from ..lib import save_channels_config, get_channels_config, import_channels_config, ChannelsConfig, \
convert_or_generate_poster
@ -25,23 +25,23 @@ def test_get_absolute_media_path(test_directory):
assert str(path).endswith('videos')
def test_get_video_duration(test_directory, corrupted_video_file):
def test_extract_video_duration(test_directory, corrupted_video_file):
"""
Video duration can be retrieved from the video file.
"""
video_path = PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4'
assert get_video_duration(video_path) == 5
assert extract_video_duration(video_path) == 5
with pytest.raises(FileNotFoundError):
get_video_duration(PROJECT_DIR / 'test/does not exist.mp4')
extract_video_duration(PROJECT_DIR / 'test/does not exist.mp4')
with pytest.raises(subprocess.CalledProcessError):
empty_file = test_directory / 'empty.mp4'
empty_file.touch()
get_video_duration(empty_file)
extract_video_duration(empty_file)
with pytest.raises(subprocess.CalledProcessError):
get_video_duration(corrupted_video_file)
extract_video_duration(corrupted_video_file)
def test_convert_image():
@ -304,10 +304,10 @@ async def test_ffprobe_json(async_client, video_file, corrupted_video_file):
@pytest.mark.asyncio
async def test_video_ffprobe_json(test_session, video_file):
async def test_video_ffprobe_json(async_client, test_session, video_file):
"""ffprobe data is extracted when a video is modeled."""
with mock.patch('modules.videos.lib.get_video_duration') as mock_get_video_duration:
mock_get_video_duration.side_effect = Exception('duration should be from ffprobe json')
with mock.patch('modules.videos.lib.extract_video_duration') as mock_extract_video_duration:
mock_extract_video_duration.side_effect = Exception('duration should be from ffprobe json')
await files_lib.refresh_files()
video = test_session.query(Video).one()

94
modules/videos/test/test_downloader.py
View file

@ -2,6 +2,7 @@ import asyncio
import json
import shutil
from copy import copy
from http import HTTPStatus
from unittest import mock
import pytest
@ -9,8 +10,10 @@ import pytest
from modules.videos.downloader import VideoDownloader, \
get_or_create_channel, channel_downloader, video_downloader
from modules.videos.models import Channel, Video
from wrolpi.conftest import test_directory
from wrolpi.downloader import Download, DownloadResult
from wrolpi.errors import InvalidDownload
from wrolpi.switches import await_switches
from wrolpi.vars import PROJECT_DIR
example_video_json = {
@ -50,6 +53,7 @@ example_channel_json = {
'uploader': None,
'url': 'video_1_url',
'view_count': 58504,
'upload_date': '20190707',
'webpage_url': 'https://youtube.com/watch?v=video_1_url'},
{'_type': 'url',
'description': None,
@ -60,6 +64,7 @@ example_channel_json = {
'uploader': None,
'url': 'video_2_url',
'view_count': 1413,
'upload_date': '20190707',
'webpage_url': 'https://youtube.com/watch?v=video_2_url'},
],
'extractor': 'youtube:tab',
@ -124,7 +129,7 @@ async def test_download_video_tags(test_session, video_download_manager, video_f
@pytest.mark.asyncio
async def test_download_channel(test_session, simple_channel, video_download_manager, video_file,
async def test_download_channel(test_session, test_directory, simple_channel, video_download_manager, video_file,
mock_video_extract_info, mock_video_prepare_filename,
mock_video_process_runner):
"""Downloading (updating the catalog of) a Channel updates its info_json.
@ -170,6 +175,7 @@ async def test_download_channel(test_session, simple_channel, video_download_man
assert downloads[0].settings == {
'channel_id': 1,
'channel_url': 'https://www.youtube.com/c/LearningSelfReliance/videos',
'destination': str(test_directory),
}
reset_downloads()
@ -187,17 +193,21 @@ async def test_download_channel(test_session, simple_channel, video_download_man
assert downloads[0].settings == {
'channel_id': 1,
'channel_url': 'https://www.youtube.com/c/LearningSelfReliance/videos',
'destination': str(test_directory),
}
def test_get_or_create_channel(test_session):
@pytest.mark.asyncio
async def test_get_or_create_channel(async_client, test_session, test_directory, tag_factory):
"""A Channel may need to be created for an arbitrary download.
Attempt to use an existing Channel if we can match it.
"""
c1 = Channel(name='foo', source_id='foo', url='foo')
one, two = await tag_factory(), await tag_factory()
c1 = Channel(name='foo', source_id='foo', url='https://example.com')
c2 = Channel(name='bar', source_id='bar')
c3 = Channel(name='baz', source_id='baz', url='baz')
c3 = Channel(name='baz', source_id='baz', url='https://example.net')
c4 = Channel(name='qux')
test_session.add_all([c1, c2, c3, c4])
test_session.commit()
@ -205,8 +215,8 @@ def test_get_or_create_channel(test_session):
# All existing channels should be used.
tests = [
(dict(source_id='foo'), c1),
(dict(url='foo'), c1),
(dict(url='foo', source_id='bar'), c2), # source_id has priority.
(dict(url='https://example.com'), c1),
(dict(url='https://example.com', source_id='bar'), c2), # source_id has priority.
(dict(name='foo', source_id='bar'), c2),
(dict(source_id='bar'), c2),
(dict(source_id='baz'), c3),
@ -217,15 +227,30 @@ def test_get_or_create_channel(test_session):
assert expected.id == channel.id, f'Expected {expected} for {kwargs} but got {channel}'
# A new channel is created. It will not be automatically downloaded.
channel = get_or_create_channel(source_id='quux', name='quux', url='quux')
channel = get_or_create_channel(source_id='quux', name='quux', url='https://example.org')
assert channel.id == 5
assert channel.source_id == 'quux'
assert channel.name == 'quux'
assert channel.url == 'quux'
assert channel.url == 'https://example.org'
# New channel can be retrieved.
assert get_or_create_channel(source_id='quux') == channel
# A Channel can be created with a tag name.
channel = get_or_create_channel(name='One Channel', tag_name=one.name)
await await_switches()
assert channel.directory == test_directory / f'videos/one/One Channel'
assert channel.tag_name == one.name and channel.tag == one
# Getting the same Channel will not change the tag.
channel = get_or_create_channel(name='One Channel', tag_name=two.name)
assert channel.directory == test_directory / f'videos/one/One Channel'
assert channel.tag_name == one.name and channel.tag == one
channel = get_or_create_channel(name='One Channel')
assert channel.directory == test_directory / f'videos/one/One Channel'
assert channel.directory.is_dir()
assert channel.tag_name == one.name and channel.tag == one
def test_channel_downloader_hidden(video_download_manager):
"""
@ -331,9 +356,8 @@ async def test_download_destination(test_session, test_directory, video_download
mock_video_prepare_filename.reset_mock()
settings = dict(destination=f'{test_directory}/custom')
video_download_manager.create_download('https://example.com/2', downloader_name=VideoDownloader.name,
settings=settings)
destination=f'{test_directory}/custom')
await video_download_manager.wait_for_all_downloads()
# Output directory matches the custom directory specified.
assert mock_video_prepare_filename.call_args_list[0].kwargs['ydl'].params['outtmpl']['default'] \
@ -356,6 +380,7 @@ example_playlist_json = {
'ie_key': 'Youtube',
'title': 'video 2 title',
'uploader': None,
'upload_date': '20241108',
'url': 'https://www.youtube.com/shorts/video_2_url',
'view_count': 1413,
'webpage_url': 'https://www.youtube.com/shorts/video_2_url'},
@ -366,6 +391,7 @@ example_playlist_json = {
'ie_key': 'Youtube',
'title': 'video 1 title',
'uploader': None,
'upload_date': '20241108',
'url': 'https://www.youtube.com/watch?v=video_1_url',
'view_count': 58504,
'webpage_url': 'https://www.youtube.com/watch?v=video_1_url'},
@ -376,6 +402,7 @@ example_playlist_json = {
'ie_key': 'Youtube',
'title': 'video 3 title',
'uploader': None,
'upload_date': '20241108',
'url': 'https://youtube.com/watch?v=video_3_url',
'view_count': 58504,
'webpage_url': 'https://youtube.com/watch?v=video_3_url'},
@ -403,7 +430,7 @@ example_playlist_json = {
async def test_download_playlist(test_session, test_directory, mock_video_extract_info, video_download_manager,
video_file):
"""All videos in a playlist can be downloaded for it's Channel."""
download = Download(url='playlist url')
download = Download(url='https://example.com/playlist-url')
test_session.add(download)
channel = get_or_create_channel(example_playlist_json['channel_id'], download.url, example_channel_json['uploader'])
video = Video.from_paths(test_session, video_file)
@ -422,3 +449,48 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac
'https://www.youtube.com/watch?v=video_2_url', # Shorts is converted to regular URL.
'https://youtube.com/watch?v=video_3_url',
}
@pytest.mark.asyncio
async def test_channel_download_crud(test_session, async_client, assert_downloads, tag_factory):
"""Test creatin"""
one, two = await tag_factory(), await tag_factory()
download = {
'urls': ['https://www.youtube.com/@wrolpi/videos'],
'destination': '',
'downloader': 'video_channel',
'sub_downloader': 'video',
'frequency': 2592000,
'tag_names': [one.name, two.name],
'settings': {
'channel_tag_name': [one.name, ],
'download_order': 'newest',
'maximum_duration': 600,
'minimum_duration': 60,
'title_exclude': 'foo,bar',
'title_include': 'baz,qux',
'video_count_limit': 100,
'video_resolutions': ['1080p', '720p', '480p', '360p', 'maximum'],
'video_format': 'mp4',
}
}
request, response = await async_client.post('/api/download', json=download)
assert response.status == HTTPStatus.NO_CONTENT
download = test_session.query(Download).one()
assert download.url == 'https://www.youtube.com/@wrolpi/videos'
assert download.tag_names == ['one', 'two']
assert download.downloader == 'video_channel'
assert download.sub_downloader == 'video'
assert download.frequency == 2592000
assert download.settings
assert download.settings['channel_tag_name'] == ['one']
assert download.settings['download_order'] == 'newest'
assert download.settings['maximum_duration'] == 600
assert download.settings['minimum_duration'] == 60
assert download.settings['title_exclude'] == 'foo,bar'
assert download.settings['title_include'] == 'baz,qux'
assert download.settings['video_count_limit'] == 100
assert download.settings['video_resolutions'] == ['1080p', '720p', '480p', '360p', 'maximum']
assert download.settings['video_format'] == 'mp4'

24
modules/videos/test/test_lib.py
View file

@ -1,5 +1,6 @@
import pathlib
import shutil
from http import HTTPStatus
import pytest
@ -266,3 +267,26 @@ async def test_format_videos_destination(async_client, test_directory):
wrolpi_config.videos_destination = '%(channel_tag)s/%(channel)s'
with pytest.raises(FileNotFoundError):
assert format_videos_destination()
@pytest.mark.asyncio
async def test_video_downloader_config_api(async_client, test_directory):
config = lib.get_downloader_config()
assert config.video_resolutions == ['1080p', '720p', '480p', '360p', 'maximum']
request, response = await async_client.get('/api/config/videos_downloader.yaml')
assert response.status == HTTPStatus.OK
config = response.json['config']
config['video_resolutions'] = ['720p', 'maximum']
body = dict(config=config)
request, response = await async_client.post('/api/config/videos_downloader.yaml', json=body)
assert response.status == HTTPStatus.NO_CONTENT
config = lib.get_downloader_config()
assert config.video_resolutions == ['720p', 'maximum']
assert (test_directory / 'config/videos_downloader.yaml').is_file()
assert config.is_valid()
contents = config.read_config_file()
assert contents['video_resolutions'] == ['720p', 'maximum']
assert '1080p' not in config.get_file().read_text()

4
requirements.txt
View file

@ -1,5 +1,5 @@
Pillow==10.4.0
Pint==0.24.3
Pint==0.24.4
PyYAML==6.0.2
SQLAlchemy[asyncio]==1.3.22
aiohttp==3.10.5
@ -28,4 +28,4 @@ srt==3.5.3
vininfo[cli]==1.8.0
websockets==13.1
webvtt-py==0.5.1
yt-dlp==2024.10.7
yt-dlp==2024.10.22

3
scripts/import_map.sh
View file

@ -106,3 +106,6 @@ fi
# Clear map tile cache only after successful import.
yes | /bin/bash /opt/wrolpi/scripts/clear_map_cache.sh
# Use curl to fetch the first few layers of map tiles so the map is ready to use.
bash /opt/wrolpi/wrolpi/scripts/initialize_map_tiles.sh

28
scripts/reset_map.sh
View file

@ -8,10 +8,20 @@ if [ $EUID != 0 ]; then
exit $?
fi
if psql -l 2>/dev/null | grep gis >/dev/null; then
# Change directory to avoid "sudo" home directory warning.
cd /tmp
if sudo -u postgres psql -l 2>/dev/null | grep gis >/dev/null; then
yes_or_no "Are you sure you want to delete the map database and cache? This can't be undone!" || exit 0
fi
function cleanup() {
sudo rm -f /tmp/tile.png /tmp/tile.md5
}
trap cleanup EXIT
cleanup
set -e
set -x
@ -25,6 +35,22 @@ sudo -u postgres psql -d wrolpi -c "UPDATE map_file SET imported=false"
yes | /opt/wrolpi/scripts/initialize_map_db.sh
# Use curl to fetch the first few layers of map tiles so the map is ready to use.
bash /opt/wrolpi/wrolpi/scripts/initialize_map_tiles.sh
# Check the hash value of a Washington DC tile from /opt/wrolpi-blobs/gis-map.dump.gz
wget "https://127.0.0.1:8084/hot/13/2345/3134.png" \
--quiet \
--no-check-certificate \
--timeout 600 \
-O /tmp/tile.png
set +x
echo "b9b7884d64b555fd4f433f991a86cc72 /tmp/tile.png" > /tmp/tile.md5
if md5sum -c /tmp/tile.md5 >/dev/null ; then
echo "Map tile was rendered correctly"
else
echo "Map tile was not rendered correctly!"
exit 1
fi
echo "Map has been reset."

5
wrolpi/api_utils.py
View file

@ -101,7 +101,10 @@ def json_error_handler(request: Request, exception: Exception):
error = repr(str(body["error"]))
message = repr(str(body["message"]))
code = body['code']
logger.error(f'API returning JSON error {type(exception).__name__} {error=} {message=} {code=}')
if logger.isEnabledFor(logging.DEBUG):
logger.error(f'API returning JSON error {type(exception).__name__} {error=} {message=} {code=}', exc_info=exception)
else:
logger.error(f'API returning JSON error {type(exception).__name__} {error=} {message=} {code=}')
if isinstance(exception, SanicException):
return json_response(body, exception.status_code)

76
wrolpi/common.py
View file

@ -1,6 +1,7 @@
import asyncio
import atexit
import contextlib
import functools
import inspect
import json
import logging
@ -40,7 +41,8 @@ from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from wrolpi.dates import now, from_timestamp, seconds_to_timestamp
from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError, InvalidConfig
from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError, InvalidConfig, \
ValidationError
from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS, LOG_LEVEL
LOGGING_CONFIG = {
@ -365,8 +367,8 @@ class ConfigFile:
from wrolpi.switches import register_switch_handler
@register_switch_handler(f'background_save_{self.file_name}')
def background_save():
self.save()
def background_save(overwrite: bool = False):
self.save(overwrite=overwrite)
self.background_save = background_save
@ -379,7 +381,7 @@ class ConfigFile:
def __repr__(self):
return f'<{self.__class__.__name__} file={self.get_file()}>'
def __json__(self) -> dict:
def config_status(self) -> dict:
d = dict(
file_name=self.file_name,
rel_path=get_relative_to_media_directory(self.get_file()),
@ -388,6 +390,9 @@ class ConfigFile:
)
return d
def __json__(self) -> dict:
return dict(self._config)
def initialize(self, multiprocessing_dict: Optional[DictProxy] = None):
"""Initializes this config dict using the default config and the config file."""
# Use the provided multiprocessing.Manager().dict(), or dict() for testing.
@ -414,7 +419,7 @@ class ConfigFile:
from wrolpi.api_utils import api_app
api_app.shared_ctx.configs_imported[self.file_name] = value
def read_config_file(self, file: pathlib.Path = None) -> Dict:
def read_config_file(self, file: pathlib.Path = None) -> dict:
file = file or self.get_file()
with file.open('rt') as fh:
config_data = yaml.load(fh, Loader=yaml.Loader)
@ -539,13 +544,16 @@ class ConfigFile:
# Copy the data as-is by default. Other classes will overwrite this and dump the database before saving.
self.save(file, send_events, overwrite)
def update(self, config: dict):
def update(self, config: dict, overwrite: bool = False):
"""Update any values of this config. Save the config to its file."""
from wrolpi.api_utils import api_app
if not self.validate(config):
raise ValidationError(f'Invalid config: {config}')
with api_app.shared_ctx.config_update_lock:
config = {k: v for k, v in config.items() if k in self._config}
self._config.update(config)
self.background_save.activate_switch()
self.background_save.activate_switch(context={'overwrite': overwrite})
def dict(self) -> dict:
"""Get a deepcopy of this config."""
@ -554,6 +562,21 @@ class ConfigFile:
return deepcopy(self._config)
def validate(self, config: dict) -> bool:
allowed_fields = {i.name for i in fields(self.validator)}
try:
# Remove keys no longer in the config.
config_items = config.items()
extra_items = {k: v for k, v in config_items if k not in allowed_fields}
if extra_items:
logger.error(f'Invalid config items: {extra_items}')
config = {k: v for k, v in config_items if k in allowed_fields}
self.validator(**config)
return True
except Exception as e:
logger.debug(f'Failed to validate config', exc_info=e)
return False
def is_valid(self, file: pathlib.Path = None) -> bool:
if not self.validator:
raise NotImplementedError(f'Cannot validate {self.file_name} without validator!')
@ -562,20 +585,13 @@ class ConfigFile:
if not file.is_file():
return False
allowed_fields = {i.name for i in fields(self.validator)}
try:
# Remove keys no longer in the config.
config_items = self.read_config_file(file).items()
extra_items = {k: v for k, v in config_items if k not in allowed_fields}
if extra_items:
logger.error(f'Invalid config items ({get_relative_to_media_directory(file)}): {extra_items}')
config = {k: v for k, v in config_items if k in allowed_fields}
self.validator(**config)
return True
except Exception as e:
logger.debug(f'Failed to validate config: {file}', exc_info=e)
config_data = self.read_config_file(file)
except InvalidConfig:
return False
return self.validate(config_data)
# `version` is used to prevent overwriting of newer configs. Cannot not be modified directly.
@property
@ -1007,7 +1023,7 @@ def check_media_directory():
return result
def get_absolute_media_path(path: str) -> Path:
def get_absolute_media_path(path: str | Path) -> Path:
"""
Get the absolute path of file/directory within the config media directory.
@ -1053,7 +1069,7 @@ def make_media_directory(path: Union[str, Path]):
path.mkdir(parents=True)
def extract_domain(url):
def extract_domain(url: str) -> str:
"""
Extract the domain from a URL. Remove leading www.
@ -2152,3 +2168,23 @@ def unique_by_predicate(
return unique_items
# Return the same type as the provided iterable
return iterable.__class__(unique_items)
def cached_multiprocessing_result(func: callable):
"""Simple multiprocessing results cacher which uses the args/kwargs of the wrapped function as the caching key.
@warning: Asumes the wrapped function is async.
"""
@functools.wraps(func)
async def wrapped(*args, **kwargs):
from wrolpi.api_utils import api_app
key = (func.__name__, *args, *tuple(kwargs.items()))
if result := api_app.shared_ctx.cache.get(key):
return result
result = await func(*args, **kwargs)
api_app.shared_ctx.cache[key] = result
return result
return wrapped

4
wrolpi/contexts.py
View file

@ -19,7 +19,7 @@ def attach_shared_contexts(app: Sanic):
# ConfigFile multiprocessing_dict's.
# Shared Configs
app.shared_ctx.wrolpi_config = multiprocessing.Manager().dict()
app.shared_ctx.wrolpi_config = manager.dict()
app.shared_ctx.tags_config = manager.dict()
app.shared_ctx.inventories_config = manager.dict()
app.shared_ctx.channels_config = manager.dict()
@ -30,6 +30,7 @@ def attach_shared_contexts(app: Sanic):
app.shared_ctx.uploaded_files = manager.dict()
app.shared_ctx.status = manager.dict()
app.shared_ctx.map_importing = manager.dict()
app.shared_ctx.cache = manager.dict()
# Shared ints
app.shared_ctx.log_level = multiprocessing.Value(ctypes.c_int, LOG_LEVEL_INT)
@ -86,6 +87,7 @@ def reset_shared_contexts(app: Sanic):
memory_stats=dict(),
))
app.shared_ctx.map_importing.clear()
app.shared_ctx.cache.clear()
# Shared ints
app.shared_ctx.log_level.value = LOG_LEVEL_INT

256
wrolpi/downloader.py
View file

@ -7,7 +7,6 @@ import os
import pathlib
import tempfile
import traceback
from abc import ABC
from dataclasses import dataclass, field
from datetime import timedelta, datetime
from enum import Enum
@ -19,7 +18,7 @@ from urllib.parse import urlparse
import feedparser
import pytz
from feedparser import FeedParserDict
from sqlalchemy import Column, Integer, String, Text, ForeignKey
from sqlalchemy import Column, Integer, String, Text, ForeignKey, ARRAY
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Session, relationship
from sqlalchemy.sql import Delete
@ -27,11 +26,12 @@ from sqlalchemy.sql import Delete
from wrolpi import flags
from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
wrol_mode_enabled, background_task, get_wrolpi_config
wrol_mode_enabled, background_task, get_wrolpi_config, get_absolute_media_path
from wrolpi.dates import TZDateTime, now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload, ValidationError
from wrolpi.events import Events
from wrolpi.media_path import MediaPathType
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod, await_switches
from wrolpi.vars import PYTEST, SIMULTANEOUS_DOWNLOAD_DOMAINS
@ -93,24 +93,23 @@ class Download(ModelHelper, Base): # noqa
url = Column(String, nullable=False, unique=True)
attempts = Column(Integer, default=0)
destination: pathlib.Path = Column(MediaPathType) # '/media/wrolpi/videos/WROLPi'
downloader = Column(Text) # 'videos', 'archive', 'kiwix_zim', etc.
sub_downloader = Column(Text) # The downloader any returned downloads should be sent.
error = Column(Text) # traceback from an error during downloading.
frequency = Column(Integer) # seconds between re-downloading.
info_json = Column(JSONB) # information retrieved WHILE downloading (from yt-dlp)
last_successful_download = Column(TZDateTime)
location = Column(Text) # Relative App URL where the item is downloaded
location = Column(Text) # Relative App URL where the downloaded item can be viewed.
next_download = Column(TZDateTime)
settings = Column(JSONB) # information about how the download should happen (destination, etc.)
settings = Column(JSONB) # information about how the download should happen (video_resolution, etc.)
status = Column(String, default=DownloadStatus.new) # `DownloadStatus` enum.
tag_names = Column(ARRAY(Text))
# A Download may be associated with a Channel (downloads all Channel videos, or a playlist, etc.).
channel_id = Column(Integer, ForeignKey('channel.id'))
channel = relationship('Channel', primaryjoin='Download.channel_id==Channel.id', back_populates='downloads')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def __repr__(self):
if self.next_download or self.frequency:
return f'<Download id={self.id} status={self.status} url={repr(self.url)} ' \
@ -125,6 +124,7 @@ class Download(ModelHelper, Base): # noqa
channel_id=self.channel_id,
downloader=self.downloader,
frequency=self.frequency,
destination=self.destination,
id=self.id,
last_successful_download=self.last_successful_download,
location=self.location,
@ -132,6 +132,7 @@ class Download(ModelHelper, Base): # noqa
settings=self.settings,
status=self.status,
sub_downloader=self.sub_downloader,
tag_names=self.tag_names,
url=self.url,
)
return d
@ -196,6 +197,8 @@ class Download(ModelHelper, Base): # noqa
def filter_excluded(self, urls: List[str]) -> List[str]:
"""Return any URLs that do not match my excluded_urls."""
if self.settings and (excluded_urls := self.settings.get('excluded_urls')):
excluded_urls = excluded_urls.split(',')
def excluded(url: str):
return any(i in url for i in excluded_urls)
@ -222,7 +225,7 @@ class Download(ModelHelper, Base): # noqa
save_downloads_config.activate_switch()
@staticmethod
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
def get_by_id(id_: int, session: Session = None) -> Optional['Download']:
download = session.query(Download).filter(Download.id == id_).one_or_none()
return download
@ -250,8 +253,7 @@ class Download(ModelHelper, Base): # noqa
@optional_session
def get_all_by_destination(cls, destination: str | pathlib.Path, session: Session = None) -> List['Download']:
destination = str(destination)
downloads = session.query(Download) \
.filter(Download.settings.contains({'destination': destination})).all() # noqa
downloads = session.query(Download).filter_by(destination=destination).all()
return downloads
@ -498,17 +500,33 @@ class DownloadManager:
@optional_session
def create_downloads(self, urls: List[str], downloader_name: str, session: Session = None,
reset_attempts: bool = False, sub_downloader_name: str = None, settings: dict = None) \
reset_attempts: bool = False, sub_downloader_name: str = None,
destination: str | pathlib.Path = None, tag_names: List[str] = None, settings: dict = None) \
-> List[Download]:
"""Schedule all URLs for download. If one cannot be downloaded, none will be added."""
if not urls or not all(urls):
raise ValueError(f'Download must have a URL: {urls=}')
logger.debug(f'Attempting to create {len(urls)} new downloads')
# Ensure all Tags exist before creating downloads.
if tag_names:
with get_db_curs() as curs:
stmt = 'SELECT name FROM tag WHERE name = ANY(%(tag_names)s)'
curs.execute(stmt, dict(tag_names=tag_names))
existing_tag_names = {i[0] for i in curs.fetchall()}
missing_tag_names = set(tag_names) - existing_tag_names
if missing_tag_names:
raise ValidationError(f'Tag does not exist: {missing_tag_names.pop()}')
downloads = []
# Throws an error if no downloader is found.
self.find_downloader_by_name(downloader_name)
if destination:
destination = pathlib.Path(destination) if isinstance(destination, str) else destination
if not destination.is_absolute():
destination = get_absolute_media_path(destination)
for url in urls:
if url in get_download_manager_config().skip_urls and reset_attempts:
# User manually entered this download, remove it from the skip list.
@ -520,9 +538,15 @@ class DownloadManager:
download = self.get_or_create_download(url, session, reset_attempts=reset_attempts)
# Download may have failed, try again.
download.renew(reset_attempts=reset_attempts)
download.destination = destination or None
download.downloader = downloader_name
download.sub_downloader = sub_downloader_name
download.tag_names = tag_names or None
# Preserve existing settings, unless new settings are provided.
download.settings = settings if settings is not None else download.settings
if download.frequency and download.settings and (channel_id := download.settings.get('channel_id')):
# Attach a recurring Channel download to it's Channel.
download.channel_id = download.channel_id or channel_id
downloads.append(download)
@ -543,16 +567,18 @@ class DownloadManager:
@optional_session
def create_download(self, url: str, downloader_name: str, session: Session = None, reset_attempts: bool = False,
sub_downloader_name: str = None, settings: Dict = None) -> Download:
sub_downloader_name: str = None, destination: str | pathlib.Path = None,
tag_names: List[str] = None, settings: Dict = None) -> Download:
"""Schedule a URL for download. If the URL failed previously, it may be retried."""
downloads = self.create_downloads([url], session=session, downloader_name=downloader_name,
reset_attempts=reset_attempts, sub_downloader_name=sub_downloader_name,
settings=settings)
destination=destination, tag_names=tag_names, settings=settings)
return downloads[0]
@optional_session
def recurring_download(self, url: str, frequency: int, downloader_name: str, session: Session = None,
sub_downloader_name: str = None, reset_attempts: bool = False,
destination: str | pathlib.Path = None, tag_names: List[str] = None,
settings: Dict = None) -> Download:
"""Schedule a recurring download."""
if not frequency or not isinstance(frequency, int):
@ -560,7 +586,7 @@ class DownloadManager:
download, = self.create_downloads([url, ], session=session, downloader_name=downloader_name,
reset_attempts=reset_attempts, sub_downloader_name=sub_downloader_name,
settings=settings)
destination=destination, tag_names=tag_names, settings=settings)
download.frequency = frequency
# Only recurring Downloads can be Channel Downloads.
@ -572,6 +598,22 @@ class DownloadManager:
return download
@optional_session
def update_download(self, id_: int, url: str, downloader: str,
destination: str | pathlib.Path = None, tag_names: List[str] = None,
sub_downloader: str | None = None,
settings: Dict = None, session: Session = None) -> Download:
download = self.find_download(session=session, id_=id_)
download.url = url
download.downloader = downloader
# Preserve existing settings, unless new settings are provided.
download.settings = settings if settings is not None else download.settings
# Use provided params even if empty.
download.destination = destination or None
download.tag_names = tag_names or None
download.sub_downloader = sub_downloader or None
return download
@wrol_mode_check
@optional_session
async def dispatch_downloads(self, session: Session = None):
@ -723,8 +765,10 @@ class DownloadManager:
renewed_count = 0
for download in recurring:
# A new download may not have a `next_download`, create it if necessary.
download.next_download = download.next_download or self.calculate_next_download(download, session=session)
if download.next_download < now_ and download.status not in (DownloadStatus.new, DownloadStatus.pending):
download.next_download = download.next_download or self.calculate_next_download(download,
session=session)
if download.next_download < now_ and download.status not in (
DownloadStatus.new, DownloadStatus.pending):
download.renew()
renewed_count += 1
@ -732,26 +776,36 @@ class DownloadManager:
self.log_debug(f'Renewed {renewed_count} recurring downloads')
session.commit()
# Save the config now that some Downloads renewed.
save_downloads_config.activate_switch()
@staticmethod
def get_downloads(session: Session) -> List[Download]:
downloads = list(session.query(Download).all())
return downloads
@staticmethod
def get_download(session: Session, url: str = None, id_: int = None) -> Optional[Download]:
def get_download(session: Session, url: str = None, id_: int = None) -> Download | None:
"""Attempt to find a Download by its URL or by its id."""
query = session.query(Download)
if url:
download = query.filter_by(url=url).one_or_none()
elif id:
elif id_:
download = query.filter_by(id=id_).one_or_none()
else:
raise RuntimeError('Cannot find download without some params.')
return download
@optional_session
def find_download(self, url: str = None, id_: int = None, session: Session = None) -> Download:
"""Find the requested Download, or raise an exception if not found.
@raise UnknownDownload: If the Download cannot be found."""
download = self.get_download(session=session, url=url, id_=id_)
if not download:
if url:
raise UnknownDownload(f'Cannot find Download with URL: {url}')
raise UnknownDownload(f'Cannot find Download with id: {id}')
return download
@optional_session
def delete_download(self, download_id: int, session: Session = None):
"""Delete a Download. Returns True if a Download was deleted, otherwise return False."""
@ -851,6 +905,8 @@ class DownloadManager:
with get_db_curs() as curs:
stmt = f'''
SELECT
channel_id,
destination,
downloader,
error,
frequency,
@ -861,8 +917,8 @@ class DownloadManager:
settings,
status,
sub_downloader,
url,
channel_id
tag_names,
url
FROM download
WHERE frequency IS NOT NULL
ORDER BY
@ -875,6 +931,7 @@ class DownloadManager:
stmt = f'''
SELECT
destination,
downloader,
error,
frequency,
@ -882,7 +939,9 @@ class DownloadManager:
last_successful_download,
location,
next_download,
settings,
status,
tag_names,
url
FROM download
WHERE frequency IS NULL
@ -1190,7 +1249,9 @@ class DownloadManagerConfig(ConfigFile):
if download.last_successful_download and not download.frequency:
# This once-download has completed, do not save it.
continue
new_downloads.append(dict(
destination = download.destination
new_download = dict(
destination=str(destination) if destination else None,
downloader=download.downloader,
frequency=download.frequency,
last_successful_download=download.last_successful_download,
@ -1198,9 +1259,11 @@ class DownloadManagerConfig(ConfigFile):
settings=download.settings,
status=download.status,
sub_downloader=download.sub_downloader,
tag_names=download.tag_names,
url=download.url,
))
get_download_manager_config().downloads = new_downloads
)
new_downloads.append(new_download)
get_download_manager_config().update({'downloads': new_downloads}, overwrite=overwrite)
except Exception as e:
message = f'Failed to save {self.get_relative_file()} config'
logger.error(message, exc_info=e)
@ -1224,12 +1287,21 @@ class DownloadManagerConfig(ConfigFile):
# Download in config already exists, update the DB record.
# The config is the source of truth.
existing.downloader = download['downloader']
existing.destination = download['destination']
existing.frequency = download['frequency']
existing.last_successful_download = download['last_successful_download']
existing.next_download = download['next_download']
existing.status = download['status']
existing.sub_downloader = download['sub_downloader']
existing.settings = download.get('settings') or dict()
if 'destination' in existing.settings:
# `destination` may be from old config, or from ChannelDownloader.
destination = existing.settings.pop('destination')
existing.destination = existing.destination or destination
if 'tag_names' in existing.settings:
# `tag_names` may be from old config, or from ChannelDownloader.
tag_names = existing.settings.pop('tag_names')
existing.tag_names = existing.tag_names or tag_names
existing.flush()
else:
logger.warning(f'Deleting Download {existing.url} because it is no longer in the config')
@ -1238,6 +1310,7 @@ class DownloadManagerConfig(ConfigFile):
for download in downloads_by_url.values():
# These downloads are new, import them.
download = Download(
destination=download['destination'],
downloader=download['downloader'],
frequency=download['frequency'],
last_successful_download=download['last_successful_download'],
@ -1245,6 +1318,7 @@ class DownloadManagerConfig(ConfigFile):
settings=download['settings'] or dict(),
status=download['status'],
sub_downloader=download['sub_downloader'],
tag_names=download['tag_names'],
url=(url := download['url']),
)
session.add(download)
@ -1340,7 +1414,7 @@ def parse_feed(url: str) -> FeedParserDict:
return feedparser.parse(url)
class RSSDownloader(Downloader, ABC):
class RSSDownloader(Downloader):
"""Downloads an RSS feed and creates new downloads for every unique link in the feed."""
name = 'rss'
pretty_name = 'RSS'
@ -1350,26 +1424,30 @@ class RSSDownloader(Downloader, ABC):
return '<RSSDownloader>'
async def do_download(self, download: Download) -> DownloadResult:
if isinstance(download.info_json, dict) and download.info_json.get('feed'):
feed = download.info_json['feed']
else:
# self.valid_url was not called by the manager, do it here.
feed: FeedParserDict = parse_feed(download.url)
if feed['bozo'] and not self.acceptable_bozo_errors(feed):
# Feed did not parse
return DownloadResult(success=False, error='Failed to parse RSS feed')
sub_downloader = download_manager.find_downloader_by_name(download.sub_downloader)
if not sub_downloader:
raise ValueError(f'Unable to find sub_downloader for {download.url}')
feed: FeedParserDict = parse_feed(download.url)
if feed['bozo'] and not self.acceptable_bozo_errors(feed):
# Feed did not parse
return DownloadResult(success=False, error='Failed to parse RSS feed')
if not isinstance(feed, dict) or not feed.get('entries'):
# RSS parsed but does not have entries.
return DownloadResult(success=False, error='RSS feed did not have any entries')
# Only download URLs that have not yet been downloaded.
urls = []
sub_downloader = download_manager.find_downloader_by_name(download.sub_downloader)
if not sub_downloader:
raise ValueError(f'Unable to find sub_downloader for {download.url}')
# Apply YT channel to the Download, if not already applied.
if yt_channel_id := feed.get('feed', dict()).get('yt_channelid'):
if not (download.location or download.channel_id):
self.apply_yt_channel(download.id, yt_channel_id)
for idx, entry in enumerate(feed['entries']):
# Filter entries using Download.settings.
entries = self.filter_entries(download, feed['entries'])
# Filter URL links.
urls = []
for idx, entry in enumerate(entries):
if url := entry.get('link'):
urls.append(url.strip())
else:
@ -1377,10 +1455,100 @@ class RSSDownloader(Downloader, ABC):
# Only download new URLs.
urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(*urls)]]
# Remove skipped URLs before duration checks. (Typically done after this by the download worker).
urls = [i for i in urls if not download_manager.is_skipped(i)]
if download.sub_downloader == 'video': # VideoDownloader
urls = await self.filter_videos(download, urls)
logger.info(f'Successfully got {len(urls)} new URLs from RSS {download.url}')
return DownloadResult(success=True, downloads=urls)
# Pass settings onto the next Downloader.
next_download_settings = dict()
settings = download.settings or dict()
if i := settings.get('video_resolutions'):
next_download_settings['video_resolutions'] = i
if i := settings.get('video_format'):
next_download_settings['video_format'] = i
if i := settings.get('destination'):
next_download_settings['destination'] = i
result = DownloadResult(
success=True,
downloads=urls,
settings=next_download_settings,
)
return result
@staticmethod
async def filter_videos(download: Download, urls: list[str]) -> list[str]:
"""Filter Video URLs by comparing the Download's settings."""
from modules.videos.downloader import fetch_video_duration
settings = download.settings or dict()
urls_before = urls.copy()
maximum_duration: int = settings.get('maximum_duration')
minimum_duration: int = settings.get('minimum_duration')
if maximum_duration or minimum_duration:
# RSS feeds do not have video duration in the XML, so use a cached function to fetch the duration of the
# linked videos for filtering.
new_urls = []
for url in urls:
# `fetch_video_duration` is cached so can be called frequently.
try:
if maximum_duration and await fetch_video_duration(url) > maximum_duration:
continue
except Exception as e:
logger.error(f'Failed to fetch duration: {url}', exc_info=e)
try:
if minimum_duration and await fetch_video_duration(url) < minimum_duration:
continue
except Exception as e:
logger.error(f'Failed to fetch duration: {url}', exc_info=e)
# Download videos even if we fail to fetch their duration.
new_urls.append(url)
urls = new_urls
if urls_before != urls:
logger.info(f'Filtered videos using min/maximum_duration from {len(urls_before)} to {len(urls)}')
return urls
@staticmethod
def filter_entries(download: Download, entries: List[dict]) -> List[dict]:
"""Filter Feed entries using Download's settings."""
# Use .lower() to ignore case.
title_exclude = (download.settings or dict()).get('title_exclude', '')
title_exclude = [i.lower() for i in title_exclude.split(',') if i]
title_include = (download.settings or dict()).get('title_include', '')
title_include = [i.lower() for i in title_include.split(',') if i]
if title_exclude or title_include:
filtered_entries = []
for entry in entries:
# Filter entries based off title.
title = entry.get('title', '').lower()
if title and title_exclude and any(i in title for i in title_exclude):
logger.info(f'RSSDownloader skipping excluded entry ({download.url}): {title}')
continue
if title and title_include and not any(i in title for i in title_include):
logger.info(f'RSSDownloader skipping non-included entry ({download.url}): {title}')
continue
filtered_entries.append(entry)
entries = filtered_entries
return entries
@staticmethod
def apply_yt_channel(download_id: int, yt_channel_id: str):
"""Get Channel that matches this Download, apply Channel information to the Download."""
with get_db_session() as session:
from modules.videos.models import Channel
channel = Channel.get_by_source_id(session, f'UC{yt_channel_id}')
if channel:
download_ = Download.get_by_id(download_id, session=session)
download_.channel = channel
download_.channel_id = channel.id
download_.location = download_.location or channel.location
session.commit()
@staticmethod
def acceptable_bozo_errors(feed):

6
wrolpi/files/downloader.py
View file

@ -33,8 +33,10 @@ class FileDownloader(Downloader, ABC):
media_directory = get_media_directory()
settings = download.settings or dict()
# We don't know where to put the file if we don't have a destination.
destination = download.settings.get('destination')
destination = download.destination or settings.get('destination')
if not destination:
raise UnrecoverableDownloadError(f'Cannot download the file without a destination')
@ -52,7 +54,7 @@ class FileDownloader(Downloader, ABC):
try:
await download_file(download.url, output_path, info)
fg = await upsert_file(output_path, tag_names=download.settings.get('tag_names'))
fg = await upsert_file(output_path, tag_names=download.tag_names)
location = fg.location
return DownloadResult(

2
wrolpi/files/lib.py
View file

@ -1546,6 +1546,8 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) ->
return
for i in range(2):
# Try multiple times because uploads happen concurrently and may conflict.
# TODO convert uploads to syncronous.
with get_db_session() as session:
file_group = FileGroup.from_paths(session, *paths)
# Re-index the contents of the file.

10
wrolpi/files/test/test_scrape_downloader.py
View file

@ -39,10 +39,9 @@ async def test_scrape_html_downloader(test_directory, test_session, test_downloa
settings = {
'depth': 1,
'suffix': '.PDF', # Suffix case is ignored.
'destination': str(destination),
}
test_download_manager.create_download('https://example.com/dir', 'scrape_html', settings=settings,
sub_downloader_name='file')
sub_downloader_name='file', destination=destination)
assert_download_urls(['https://example.com/dir', ])
with mock.patch('wrolpi.scrape_downloader.ScrapeHTMLDownloader.fetch_html', fake_fetch_html), \
@ -63,12 +62,9 @@ async def test_scrape_html_downloader_html(test_directory, test_session, test_do
test_download_manager.register_downloader(ScrapeHTMLDownloader())
test_download_manager.register_downloader(FileDownloader())
settings = {
'suffix': '.html',
'destination': str(test_directory),
}
settings = {'suffix': '.html'}
test_download_manager.create_download('https://example.com/dir', 'scrape_html', settings=settings,
sub_downloader_name='file')
sub_downloader_name='file', destination=test_directory)
assert_download_urls(['https://example.com/dir', ])
with mock.patch('wrolpi.scrape_downloader.ScrapeHTMLDownloader.fetch_html', fake_fetch_html), \

80
wrolpi/root_api.py
View file

@ -26,9 +26,10 @@ from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, get_medi
set_global_log_level, get_relative_to_media_directory, search_other_estimates, get_all_configs, \
get_config_by_file_name
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, HotspotError, InvalidDownload, \
HotspotPasswordTooShort, NativeOnly, InvalidConfig
HotspotPasswordTooShort, NativeOnly, InvalidConfig, ValidationError
from wrolpi.events import get_events, Events
from wrolpi.files import files_bp
from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count
@ -262,13 +263,14 @@ async def post_download(_: Request, body: schema.DownloadRequest):
if not downloader:
raise InvalidDownload(f'Cannot find downloader with name {body.downloader}')
kwargs = dict(downloader_name=body.downloader,
sub_downloader_name=body.sub_downloader, reset_attempts=True,
destination=body.destination, tag_names=body.tag_names,
settings=body.settings)
if body.frequency:
download_manager.recurring_download(body.urls[0], body.frequency, downloader_name=body.downloader,
sub_downloader_name=body.sub_downloader, reset_attempts=True,
settings=body.settings)
download_manager.recurring_download(body.urls[0], body.frequency, **kwargs)
else:
download_manager.create_downloads(body.urls, downloader_name=body.downloader, reset_attempts=True,
sub_downloader_name=body.sub_downloader, settings=body.settings)
download_manager.create_downloads(body.urls, **kwargs)
if download_manager.disabled.is_set() or download_manager.stopped.is_set():
# Downloads are disabled, warn the user.
Events.send_downloads_disabled('Download created. But, downloads are disabled.')
@ -283,10 +285,21 @@ async def post_download(_: Request, body: schema.DownloadRequest):
async def put_download(_: Request, download_id: int, body: schema.DownloadRequest):
downloader = download_manager.find_downloader_by_name(body.downloader)
if not downloader:
raise InvalidDownload(f'Cannot find downloader with name {body.downloader}')
raise ValidationError(f'Cannot find downloader with name {body.downloader}')
if len(body.urls) != 1:
raise ValidationError('Only one URL can be specified when updating a Download')
download_manager.create_downloads(body.urls, downloader_name=body.downloader, reset_attempts=True,
sub_downloader_name=body.sub_downloader, settings=body.settings)
with get_db_session(commit=True) as session:
download_manager.update_download(
id_=download_id,
url=body.urls[0],
downloader=body.downloader,
destination=body.destination,
tag_names=body.tag_names,
sub_downloader=body.sub_downloader,
settings=body.settings,
session=session,
)
if download_manager.disabled.is_set() or download_manager.stopped.is_set():
# Downloads are disabled, warn the user.
Events.send_downloads_disabled('Download created. But, downloads are disabled.')
@ -310,9 +323,18 @@ async def restart_download(_: Request, download_id: int):
@api_bp.get('/download')
@openapi.description('Get all Downloads that need to be processed.')
@openapi.description('Get all Downloads so they can be displayed to the User.')
async def get_downloads(_: Request):
data = download_manager.get_fe_downloads()
# Convert `destination` to relative.
for download in data['once_downloads']:
download['destination'] = get_relative_to_media_directory(download['destination']) \
if download['destination'] else None
for download in data['recurring_downloads']:
download['destination'] = get_relative_to_media_directory(download['destination']) \
if download['destination'] else None
return json_response(data)
@ -417,7 +439,7 @@ async def throttle_off(_: Request):
@api_bp.get('/status')
@openapi.description('Get the status of CPU/load/etc.')
async def get_status(_: Request):
async def get_status(request: Request):
downloads = dict()
if flags.db_up.is_set():
try:
@ -425,15 +447,25 @@ async def get_status(_: Request):
except Exception as e:
logger.debug('Unable to get download status', exc_info=e)
sanic_workers = dict()
if hasattr(request.app, 'multiplexer'):
# `multiplexer` may be empty while testing.
sanic_workers = {
i: {'pid': j['pid'], 'state': j['state']}
for i, j in request.app.multiplexer.workers.items()
if i.startswith('Sanic-Server')
}
ret = dict(
dockerized=DOCKERIZED,
is_rpi=IS_RPI,
is_rpi4=IS_RPI4,
is_rpi5=IS_RPI5,
downloads=downloads,
flags=flags.get_flags(),
hotspot_status=admin.hotspot_status().name,
hotspot_ssid=admin.get_current_ssid(get_wrolpi_config().hotspot_device),
hotspot_status=admin.hotspot_status().name,
is_rpi4=IS_RPI4,
is_rpi5=IS_RPI5,
is_rpi=IS_RPI,
sanic_workers=sanic_workers,
throttle_status=admin.throttle_status().name,
version=__version__,
wrol_mode=wrol_mode_enabled(),
@ -607,11 +639,29 @@ async def post_search_other_estimates(_: Request, body: schema.SearchOtherEstima
@api_bp.get('/configs')
@openapi.description('Get the status (imported, valid, etc.) of all configs.')
def get_configs(_: Request):
configs = get_all_configs()
configs = {k: v.config_status() for k, v in configs.items()}
return json_response(dict(configs=configs))
@api_bp.get('/config/<name:str>')
@openapi.description('Get the contents of a specific config.')
def get_config(_: Request, name: str):
config = get_all_configs()[name]
return json_response(dict(config=config))
@api_bp.post('/config/<name:str>')
@openapi.description('Save the contents of a specific config.')
@validate(json=schema.ConfigSaveRequest)
def save_config(_: Request, name: str, body: schema.ConfigSaveRequest):
config = get_all_configs()[name]
config.update(body.config)
return response.empty()
@api_bp.post('/configs/import')
@validate(json=schema.ConfigsImportRequest)
def post_configs_import(_: Request, body: schema.ConfigsImportRequest):

53
wrolpi/schema.py
View file

@ -1,7 +1,9 @@
import enum
import pathlib
from dataclasses import dataclass, field
from typing import Optional, List
from wrolpi.common import get_relative_to_media_directory, unique_by_predicate
from wrolpi.errors import InvalidDownload, ValidationError
@ -78,24 +80,31 @@ class EchoResponse:
@dataclass
class DownloadSettings:
channel_id: int = None
channel_tag_name: List[str] = field(default_factory=lambda: list())
depth: Optional[int] = None
destination: Optional[str] = None
download_metadata_only: bool = False
download_order: Optional[str] = None
excluded_urls: Optional[str] = None
max_pages: Optional[int] = None
maximum_duration: Optional[int] = None
minimum_duration: Optional[int] = None
suffix: Optional[str] = None
tag_names: List[str] = field(default_factory=lambda: list())
title_exclude: str = None
title_include: str = None
video_count_limit: Optional[int] = None
video_format: str = None
video_resolutions: List[str] = field(default_factory=lambda: list())
def __post_init__(self):
from wrolpi.common import get_media_directory
self.tag_names = [i.strip() for i in self.tag_names] if self.tag_names else []
if isinstance(self.excluded_urls, str):
self.excluded_urls = [i.strip() for i in self.excluded_urls.split(',')] if self.excluded_urls else None
else:
self.excluded_urls = [i.strip() for i in self.excluded_urls] if self.excluded_urls else None
self.destination = str(get_media_directory() / self.destination) if self.destination else None
if self.excluded_urls and self.excluded_urls.endswith(','):
raise ValidationError('Excluded urls cannot end with ,')
if self.excluded_urls and self.excluded_urls.startswith(','):
raise ValidationError('Excluded urls cannot start with ,')
if self.suffix and not self.suffix.startswith('.'):
raise ValidationError('suffix must start with .')
self.title_exclude = self.title_exclude or None
@ -103,24 +112,43 @@ class DownloadSettings:
if not self.download_metadata_only:
del self.download_metadata_only
if self.download_order not in (None, 'newest', 'oldest', 'views'):
raise ValidationError(f'Download order must be one of newest, oldest, views, or null.')
if self.video_format not in (None, 'mp4', 'mkv'):
raise ValidationError(f'Download order must be one of mp4, mkv, or null.')
valid_resolutions = ('360p', '480p', '720p', '1080p', '1440p', '2160p', 'maximum')
if not all(i in valid_resolutions for i in self.video_resolutions):
raise ValidationError(f'Download order must be one of 360p, 480p, 720p, 1080p, 1440p, 2160p, or maximum.')
@dataclass
class DownloadRequest:
urls: List[str]
downloader: str
destination: Optional[str] = None
tag_names: List[str] = field(default_factory=list)
frequency: Optional[int] = None
sub_downloader: Optional[str] = None
settings: Optional[dict] = field(default_factory=lambda: dict())
settings: Optional[dict] = field(default_factory=dict)
def __post_init__(self):
urls = [j for i in self.urls if (j := i.strip())]
if not urls:
raise InvalidDownload(f'urls cannot be empty')
# Get unique URLs, preserve order.
self.urls = list(dict.fromkeys(urls))
self.urls = unique_by_predicate(urls)
if self.destination:
destination = pathlib.Path(self.destination)
if destination.is_absolute():
destination = get_relative_to_media_directory(destination)
self.destination = str(destination)
# Validate settings contents. Remove empty values.
self.settings = {k: v for k, v in DownloadSettings(**self.settings).__dict__.items() if v not in ([], None)}
settings = self.settings or dict()
self.settings = {k: v for k, v in DownloadSettings(**settings).__dict__.items() if v not in ([], None)}
@dataclass
@ -219,3 +247,8 @@ class SearchOtherEstimateRequest:
class ConfigsImportRequest:
file_name: str
overwrite: bool = False
@dataclass
class ConfigSaveRequest:
config: dict

9
wrolpi/scrape_downloader.py
View file

@ -1,4 +1,5 @@
import pathlib
from copy import copy
from urllib.parse import urlparse
from wrolpi.common import logger, get_html_soup, aiohttp_get
@ -51,7 +52,7 @@ class ScrapeHTMLDownloader(Downloader):
depth = download.settings.get('depth') or 1
suffix = download.settings.get('suffix')
max_pages = download.settings.get('max_pages') or 100
destination = download.settings.get('destination')
destination = download.destination
if 0 > depth > 10:
raise UnrecoverableDownloadError('Depth must be between 0 and 100')
@ -100,8 +101,10 @@ class ScrapeHTMLDownloader(Downloader):
child_url = resolve_url(url, href)
if child_url and child_url.lower().endswith(suffix):
# Found a file that the User requested.
logger.info(f'ScrapeHTMLDownloader will download {child_url}')
download_urls.append(child_url)
else:
logger.debug(f'ScrapeHTMLDownloader scraping {child_url}')
urls.append(child_url)
if not download_urls:
@ -110,10 +113,12 @@ class ScrapeHTMLDownloader(Downloader):
error=f'No files with {suffix} found in {page_count} pages!'
)
settings = copy(download.settings)
settings['destination'] = str(download.destination) # Use str for json conversion.
return DownloadResult(
success=True,
downloads=download_urls,
settings=download.settings,
settings=settings,
error='Reached max page count.' if page_count >= max_pages else None,
location=f'/files?folders={destination}'
)

62
wrolpi/scripts/initialize_map_tiles.sh Normal file
View file

@ -0,0 +1,62 @@
#! /usr/bin/env bash
# Use curl to fetch the first few layers of map tiles so the map is ready to use.
HOST="https://127.0.0.1:8084"
Help() {
# Display Help
echo "Fetch several lays of map tiles to initialize map tile data."
echo
echo "Syntax: initialize_map_tiles.sh [-h] [-H HOST]"
echo "options:"
echo "h Print this help."
echo "b The hostname to fetch tiles from. (default: ${HOST})"
echo
}
while getopts ":hH:" option; do
case $option in
h) # display Help
Help
exit
;;
H)
HOST="${OPTARG}"
;;
*) # invalid argument(s)
echo "Error: Invalid option"
exit 1
;;
esac
done
# Try to fetch the first tile multiple times if the service is starting, or if new map was imported.
for i in {1..100}; do
# Try to fetch the first tile.
wget "${HOST}/hot/3/0/0.png" \
--quiet \
--tries=1 \
--no-check-certificate \
--timeout 5 -O /dev/null || {
echo "Attempt $i failed. Sleeping for 5 seconds..."
sleep 5
continue
}
# If we reach here, wget succeeded, so we exit the loop
break
done
# If we've made it through all attempts, curl never succeeded
if [[ $i -eq $ATTEMPTS ]]; then
echo "Failed to fetch the image after $ATTEMPTS attempts."
exit 1
fi
echo "Fetching first map tile succeeded."
curl -ks \
"${HOST}/hot/3/{0..7}/{0..7}.png" \
"${HOST}/hot/4/{0..15}/{0..15}.png" \
"${HOST}/hot/5/{0..31}/{0..31}.png" \
"${HOST}/hot/6/{0..41}/{0..41}.png" >/dev/null 2>&1
echo "Fetching many map tiles completed."

18
wrolpi/switches.py
View file

@ -1,5 +1,6 @@
import asyncio
import inspect
import logging
import multiprocessing
from functools import partial
from typing import Dict, Mapping, Protocol
@ -7,6 +8,8 @@ from typing import Dict, Mapping, Protocol
from wrolpi.api_utils import api_app, logger, perpetual_signal
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
SWITCH_HANDLERS: Dict[str, callable] = dict()
@ -33,7 +36,12 @@ def activate_switch(switch_name: str, context: dict = None):
switches_changed.set()
switches.update({**switches.copy(), switch_name: context})
logger.debug(f'activate_switch: {switch_name}')
if logger.isEnabledFor(logging.DEBUG):
# Switches can be difficult to troubleshoot. Log the function that activates a switch.
caller_frame_record = inspect.stack()[1]
frame = caller_frame_record[0]
info = inspect.getframeinfo(frame)
logger.debug(f'activate_switch: {switch_name} called by {info.function} in {info.filename}')
finally:
switches_lock.release()
else:
@ -94,15 +102,19 @@ async def switch_worker():
switches_changed.wait(timeout=1)
switches: dict = api_app.shared_ctx.switches
with api_app.shared_ctx.switches_lock:
switches_keys = switches.keys()
switch_name, context = switches.popitem()
switches_keys = list(switches.keys())
# Call handler with the stored context, await coroutine, if any.
handler = SWITCH_HANDLERS[switch_name]
logger.debug(f'switch_worker handling {switch_name} of {len(switches_keys)}')
if logger.isEnabledFor(logging.DEBUG):
logger.debug(f'switch_worker handling {switch_name} of {switches_keys}')
else:
logger.info(f'switch_worker handling {switch_name} of {len(switches_keys)}')
coro = handler(**context)
if inspect.iscoroutine(coro):
# Handler is async.
await coro
logger.debug(f'switch_worker completed {switch_name}')
except TimeoutError:
# `switches_changed` is not set.
pass

35
wrolpi/tags.py
View file

@ -13,6 +13,7 @@ from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_direc
get_relative_to_media_directory, is_valid_hex_color, walk, INVALID_FILE_CHARS
from wrolpi.dates import TZDateTime
from wrolpi.db import optional_session, get_db_curs, get_db_session
from wrolpi.downloader import save_downloads_config
from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, FileGroupAlreadyTagged, RefreshConflict, NoPrimaryFile
from wrolpi.events import Events
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
@ -182,6 +183,25 @@ class Tag(ModelHelper, Base):
from modules.videos.errors import ChannelDirectoryConflict
async def _():
# Rename old name referenced in any Downloads.
from wrolpi.downloader import Download
to_flush = list()
for download in session.query(Download):
if download.tag_names:
download.tag_names = [i if i != old_name else name for i in download.tag_names]
to_flush.append(download)
if download.settings and 'channel_tag_name' in download.settings:
settings = download.settings
tag_name = settings.pop('channel_tag_name')
tag_name = [i for i in tag_name if i != old_name]
settings['channel_tag_name'] = tag_name
download.settings = settings
to_flush.append(download)
if to_flush:
session.flush(to_flush)
save_downloads_config.activate_switch()
for channel in self.channels:
channel: Channel
possible_directory = channel.format_directory(old_name)
@ -212,8 +232,22 @@ class Tag(ModelHelper, Base):
if self.has_relations():
raise UsedTag(f'Cannot delete {self.name} it is used')
name = self.name
session = Session.object_session(self)
session.delete(self)
# Remove any Download references.
from wrolpi.downloader import Download
to_flush = list()
for download in session.query(Download):
if download.tag_names:
download.tag_names = [i for i in download.tag_names if i != name]
to_flush.append(download)
if to_flush:
session.flush(to_flush)
save_downloads_config.activate_switch()
session.commit()
save_tags_config.activate_switch()
@ -395,6 +429,7 @@ class TagsConfig(ConfigFile):
logger.debug(f'Creating TagFile for tag_id={tag.id} file_group_id={file_group.id}')
tag_file = TagFile(file_group_id=file_group.id, tag_id=tag.id, tag=tag,
file_group=file_group)
tag_files[(tag_file.tag_id, tag_file.file_group_id)] = tag_file
session.add(tag_file)
tag_file.flush()
tag_file.created_at = dates.strptime_ms(created_at) if created_at else dates.now()

25
wrolpi/test/common.py
View file

@ -1,3 +1,4 @@
import json
import pathlib
import tempfile
from contextlib import contextmanager
@ -8,6 +9,7 @@ from typing import List
import mock
import pytest
from wrolpi.api_utils import CustomJSONEncoder
from wrolpi.common import get_media_directory
from wrolpi.conftest import test_db, test_client # noqa
from wrolpi.db import postgres_engine
@ -147,11 +149,20 @@ def assert_dict_contains(d1: dict, d2: dict):
if hasattr(d2, '__dict__'):
d2 = d2.__dict__
# Sort items so error is easier to read.
d1 = dict(sorted(d1.items()))
d2 = dict(sorted(d2.items()))
for k2 in d2.keys():
assert d1, f'dict 1 is empty: {d1}'
assert d2, f'dict 1 is empty: {d2}'
assert k2 in d1, f'dict 1 does not contain {k2}'
if isinstance(d1[k2], dict):
assert_dict_contains(d1[k2], d2[k2])
else:
assert d1[k2] == d2[k2], f'{repr(k2)} of value {repr(d1[k2])} does not equal {repr(d2[k2])} in dict 1'
try:
assert d1, f'dict 1 is empty: {d1}'
assert d2, f'dict 1 is empty: {d2}'
assert k2 in d1, f'dict 1 does not contain {k2}'
if isinstance(d1[k2], dict):
assert_dict_contains(d1[k2], d2[k2])
else:
assert d1[k2] == d2[k2], f'{repr(k2)} of value {repr(d1[k2])} does not equal {repr(d2[k2])} in dict 1'
except AssertionError as e:
d1_str = json.dumps(d1, indent=1, cls=CustomJSONEncoder)
d2_str = json.dumps(d2, indent=1, cls=CustomJSONEncoder)
raise AssertionError(f'{d1_str}\n\n{d2_str}\n\nDictionaries above are unequal') from e

23
wrolpi/test/test_common.py
View file

@ -1005,6 +1005,7 @@ async def test_config_valid(async_client, test_wrolpi_config):
test_wrolpi_config.write_text('bad config')
assert config.is_valid() is False
# Does not require all items.
test_wrolpi_config.write_text('wrol_mode: true')
assert config.is_valid() is True
@ -1033,3 +1034,25 @@ async def test_config_valid(async_client, test_wrolpi_config):
])
def test_trim_file_name(name, expected_name):
assert common.trim_file_name(name) == expected_name
async def test_cached_multiprocessing_result(async_client):
call_count = 0
@common.cached_multiprocessing_result
async def func(inc: int) -> int:
nonlocal call_count
call_count += 1
return inc + call_count
# First call, math is done.
assert await func(1) == 2
# Second call is cached.
assert await func(1) == 2
assert await func(1) == 2
# New args, math is done again.
assert await func(2) == 4
assert await func(2) == 4
# Kwargs are supported
assert await func(inc=1) == 4
assert await func(inc=1) == 4

50
wrolpi/test/test_downloader.py
View file

@ -16,7 +16,7 @@ from wrolpi.common import get_wrolpi_config
from wrolpi.dates import Seconds
from wrolpi.db import get_db_context
from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \
RSSDownloader, get_download_manager_config
get_download_manager_config, RSSDownloader
from wrolpi.errors import InvalidDownload, WROLModeEnabled
from wrolpi.switches import await_switches
from wrolpi.test.common import assert_dict_contains
@ -85,6 +85,29 @@ async def test_recreate_download(test_session, test_download_manager, test_downl
assert download.settings == dict()
@pytest.mark.asyncio
async def test_download_rename_tag(async_client, test_session, test_download_manager, test_downloader, tag_factory):
"""Renaming a Tag renames any Downloads that reference that Tag."""
tag = await tag_factory()
tag_names = [tag.name, ]
test_download_manager.create_downloads(
['https://example.com/1', ], test_downloader.name, tag_names=tag_names)
download = test_session.query(Download).one()
assert download.tag_names == ['one', ]
await tag.update_tag('new name', tag.color, test_session)
download = test_session.query(Download).one()
assert download.tag_names == ['new name', ]
tag.delete()
download = test_session.query(Download).one()
assert download.tag_names == []
def test_downloader_must_have_name():
"""
A Downloader class must have a name.
@ -366,7 +389,7 @@ async def test_crud_download(async_client, test_session, test_download_manager,
assert download.url == 'https://example.com'
assert download.downloader == test_downloader.name
assert download.frequency == DownloadFrequency.weekly
assert download.settings['excluded_urls'] == ['example.org', 'something']
assert download.settings['excluded_urls'] == 'example.org,something'
request, response = await async_client.get('/api/download')
assert response.status_code == HTTPStatus.OK
@ -391,13 +414,15 @@ async def test_crud_download(async_client, test_session, test_download_manager,
@pytest.mark.asyncio
async def test_downloads_config(test_session, test_download_manager, test_download_manager_config,
test_downloader, assert_downloads):
test_downloader, assert_downloads, tag_factory):
# Can import with an empty config.
await import_downloads_config()
assert_downloads([])
test_downloader.set_test_success()
tag = await tag_factory()
download1, download2 = test_download_manager.create_downloads([
'https://example.com/1',
'https://example.com/2',
@ -405,7 +430,8 @@ async def test_downloads_config(test_session, test_download_manager, test_downlo
test_download_manager.recurring_download('https://example.com/3', frequency=DownloadFrequency.weekly,
downloader_name=test_downloader.name)
download2.next_download = datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC)
download2.settings = {'destination': 'some directory'}
download2.destination = 'some directory'
download2.tag_names = [tag.name, ]
# Completed once-downloads should be ignored.
download1.last_successful_download = datetime(2000, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
test_session.commit()
@ -422,8 +448,9 @@ async def test_downloads_config(test_session, test_download_manager, test_downlo
frequency=None,
next_download=datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC),
downloader='test_downloader',
settings={'destination': 'some directory'},
destination='some directory',
sub_downloader=None,
tag_names=['one', ]
),
dict(
url='https://example.com/3',
@ -431,6 +458,7 @@ async def test_downloads_config(test_session, test_download_manager, test_downlo
downloader='test_downloader',
settings=None,
sub_downloader=None,
tag_names=None,
),
]
for download, expected in zip_longest(get_download_manager_config().downloads, expected):
@ -440,12 +468,13 @@ async def test_downloads_config(test_session, test_download_manager, test_downlo
test_session.query(Download).delete()
test_session.commit()
# Change the destination.
# Change the destination, remove Tags.
with test_download_manager_config.open('rt') as fh:
config_contents = yaml.load(fh, Loader=yaml.Loader)
for idx, download in enumerate(config_contents['downloads']):
if download['url'] == 'https://example.com/2':
config_contents['downloads'][idx]['settings'] = dict(destination='a different directory')
config_contents['downloads'][idx]['destination'] = 'a different directory'
config_contents['downloads'][idx]['tag_names'] = None
with test_download_manager_config.open('wt') as fh:
yaml.dump(config_contents, fh)
get_download_manager_config().initialize()
@ -459,15 +488,18 @@ async def test_downloads_config(test_session, test_download_manager, test_downlo
next_download=datetime(2000, 1, 2, 0, 0, 0, tzinfo=pytz.UTC),
downloader='test_downloader',
# The destination was imported from the config.
settings={'destination': 'a different directory'},
destination=pathlib.Path('a different directory'),
sub_downloader=None,
tag_names=None, # Tags were removed.
),
dict(
url='https://example.com/3',
frequency=DownloadFrequency.weekly,
downloader='test_downloader',
destination=None,
settings=dict(),
sub_downloader=None,
tag_names=None, # Tags were not added.
),
]
assert_downloads(expected)
@ -495,7 +527,7 @@ async def test_download_excluded_urls(test_session, test_download_manager, test_
dict(link='https://example.gov/d'),
]
)
settings = dict(excluded_urls=['example.org', 'example.gov'])
settings = dict(excluded_urls='example.org,example.gov')
feed_download: Download = test_download_manager.create_download('https://example.com/feed', rss_downloader.name,
test_session,
sub_downloader_name=test_downloader.name,

30
wrolpi/test/test_root_api.py
View file

@ -398,7 +398,7 @@ async def test_retry_downloads(test_session, async_client, test_wrolpi_config, a
dict(url='https://example.com/3', status='new', attempts=0), # deferred is retried.
dict(url='https://example.com/4', status='new', attempts=1), # new does not need to be retried.
dict(url='https://example.com/5', status='failed', attempts=1), # failed has been given up on.
dict(url='https://example.com/6', status='failed', frequency=60, attempts=1,), # recurring is always retried.
dict(url='https://example.com/6', status='failed', frequency=60, attempts=1, ), # recurring is always retried.
dict(url='https://example.com/7', status='complete', frequency=60, attempts=1), # recurring is always retried.
])
@ -417,6 +417,7 @@ async def test_get_status(async_client, test_session):
assert isinstance(response.json.get('version'), str), 'version should be a str'
assert isinstance(response.json.get('memory_stats'), dict), 'memory_stats should be a dict'
assert isinstance(response.json.get('flags'), dict), 'flags should be a dict'
assert isinstance(response.json.get('sanic_workers'), dict), 'Sanic worker status should be a dict'
@pytest.mark.asyncio
@ -513,6 +514,33 @@ async def test_restart_download(test_session, async_client, test_download_manage
assert download.is_deferred, download.status
@pytest.mark.asyncio
async def test_put_download(test_session, async_client, test_download_manager, test_downloader, channel_factory):
"""A Download can be updated using a PUT request."""
channel = channel_factory()
download = test_download_manager.create_download('https://example.com', test_downloader.name)
assert not download.settings
body = dict(
downloader=test_downloader.name,
urls=['https://example.com/new-url'], # URL can be changed.
settings=dict(channel_id=channel.id),
)
request, response = await async_client.put(f'/api/download/{download.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Download).count() == 1, 'Download should have been updated'
download = test_session.query(Download).one()
assert download.url == 'https://example.com/new-url', 'Download URL was not changed'
assert download.settings == dict(channel_id=channel.id), 'Download settings were not changed'
# Settings can be cleared by sending `None`
body['settings'] = None
request, response = await async_client.put(f'/api/download/{download.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT
download = test_session.query(Download).one()
assert download.settings == dict(), 'Download settings were not changed'
def test_get_global_statistics(test_session, test_client):
request, response = test_client.get('/api/statistics')
assert response.json['global_statistics']['db_size'] > 1

31
wrolpi/test/test_rss.py
View file

@ -5,7 +5,7 @@ import mock
import pytest
from wrolpi.db import optional_session
from wrolpi.downloader import Download, DownloadResult, RSSDownloader, Downloader, DownloadManager
from wrolpi.downloader import Download, DownloadResult, Downloader, DownloadManager, RSSDownloader
class RSSHTTPDownloader(Downloader):
@ -85,6 +85,8 @@ async def test_rss_no_entries(test_session, test_download_manager):
"""An RSS feed with no entries is handled."""
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
http_downloader = RSSHTTPDownloader()
test_download_manager.register_downloader(http_downloader)
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
mock_parse_feed.return_value = dict(bozo=0, ) # missing `entries`
@ -95,3 +97,30 @@ async def test_rss_no_entries(test_session, test_download_manager):
(download,) = test_download_manager.get_downloads(test_session)
assert download.is_deferred
assert 'entries' in download.error
@pytest.mark.asyncio
async def test_rss_downloader_filter_titles(test_session):
rss_downloader = RSSDownloader()
entries = [
dict(link='https://example.com/a', title='A'), # Case should be ignored.
dict(link='https://example.com/b', title='b'),
dict(link='https://example.com/c', title='c'),
]
download = Download(url='https://example.com/feed', settings=dict(title_include='a'))
assert rss_downloader.filter_entries(download, entries) == [
dict(link='https://example.com/a', title='A'),
]
download = Download(url='https://example.com/feed', settings=dict(title_include='b'))
assert rss_downloader.filter_entries(download, entries) == [
dict(link='https://example.com/b', title='b'),
]
download = Download(url='https://example.com/feed', settings=dict(title_exclude='C'))
assert rss_downloader.filter_entries(download, entries) == [
dict(link='https://example.com/a', title='A'),
dict(link='https://example.com/b', title='b'),
]

2
wrolpi/test/test_status.py
View file

@ -4,6 +4,7 @@ from unittest import mock
import pytest
from wrolpi import status
from wrolpi.test.common import skip_circleci
@pytest.mark.asyncio
@ -26,6 +27,7 @@ async def test_get_cpu_stats(async_client):
assert isinstance(info.temperature, int)
@skip_circleci
@pytest.mark.asyncio
async def test_get_drives_stats(async_client):
"""Minimum drives info testing because this will fail in docker, etc."""