mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
Creating RecursiveHTMLDownloader which recursively searches for files in the provided URLs.
This commit is contained in:
parent
1b4d4d7ad5
commit
6f43f1daa0
12 changed files with 377 additions and 15 deletions
|
|
@ -439,7 +439,18 @@ export async function getArchive(archiveId) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function postDownload(urls, downloader, frequency, sub_downloader, excludedURLs, destination, tagNames) {
|
||||
export async function postDownload(
|
||||
urls,
|
||||
downloader,
|
||||
frequency,
|
||||
sub_downloader,
|
||||
excludedURLs,
|
||||
destination,
|
||||
tagNames,
|
||||
depth,
|
||||
suffix,
|
||||
max_pages,
|
||||
) {
|
||||
if (!downloader) {
|
||||
toast({
|
||||
type: 'error',
|
||||
|
|
@ -456,6 +467,9 @@ export async function postDownload(urls, downloader, frequency, sub_downloader,
|
|||
frequency: frequency || null,
|
||||
excluded_urls: excludedURLs,
|
||||
destination: destination || null,
|
||||
depth: depth || null,
|
||||
suffix: suffix || null,
|
||||
max_pages: max_pages || null,
|
||||
};
|
||||
if (sub_downloader) {
|
||||
body['sub_downloader'] = sub_downloader;
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {AccordionContent, AccordionTitle, Form as SForm, FormDropdown} from "sem
|
|||
import {Link} from "react-router-dom";
|
||||
import {TagsSelector} from "../Tags";
|
||||
import {toast} from "react-semantic-toasts-2";
|
||||
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
|
||||
|
||||
const validUrl = /^(http|https):\/\/[^ "]+$/;
|
||||
|
||||
|
|
@ -429,6 +430,147 @@ class RSSDownload extends ChannelDownload {
|
|||
}
|
||||
}
|
||||
|
||||
class RecursiveDownload extends Downloader {
|
||||
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.state = {
|
||||
depth: 1,
|
||||
disabled: false,
|
||||
max_pages: 100,
|
||||
pending: false,
|
||||
ready: false,
|
||||
sub_downloader: null,
|
||||
success: null,
|
||||
suffix: '',
|
||||
urls: '',
|
||||
};
|
||||
}
|
||||
|
||||
handleInputChange = async (event, {name, value}) => {
|
||||
this.setState({[name]: value});
|
||||
}
|
||||
|
||||
submitDownload = async () => {
|
||||
let {urls, destination, depth, suffix, max_pages} = this.state;
|
||||
if (urls) {
|
||||
this.setState({pending: true, submitted: false});
|
||||
try {
|
||||
let response = await postDownload(
|
||||
urls,
|
||||
'recursive_html',
|
||||
null,
|
||||
'file',
|
||||
null,
|
||||
destination,
|
||||
null,
|
||||
depth,
|
||||
suffix,
|
||||
max_pages,
|
||||
);
|
||||
if (response.status === 204) {
|
||||
this.setState({urls: '', pending: false, submitted: true});
|
||||
} else {
|
||||
const content = await response.json();
|
||||
let error = 'Failed to create download!';
|
||||
if (content && content['error']) {
|
||||
error = content['error'];
|
||||
}
|
||||
toast({type: 'error', title: 'Error', description: error});
|
||||
}
|
||||
} finally {
|
||||
this.setState({pending: false});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
render() {
|
||||
const {urls, depth, max_pages, suffix, destination} = this.state;
|
||||
const depths = [
|
||||
{key: 1, text: 1, value: 1},
|
||||
{key: 2, text: 2, value: 2},
|
||||
{key: 3, text: 3, value: 3},
|
||||
{key: 4, text: 4, value: 4},
|
||||
];
|
||||
const validSuffix = suffix && suffix.startsWith('.');
|
||||
const complete = urls && depth && validSuffix && max_pages && destination;
|
||||
|
||||
return <Form>
|
||||
<Header as='h4'><Icon name='file alternate' color='red'/> Recursive File Downloader</Header>
|
||||
|
||||
<p>Search each of the URLs for files matching the suffix (.pdf, etc.).</p>
|
||||
|
||||
<TextArea required
|
||||
placeholder='Enter one URL per line'
|
||||
name='urls'
|
||||
onChange={this.handleInputChange}
|
||||
value={urls}
|
||||
style={{marginBottom: '1em'}}
|
||||
/>
|
||||
|
||||
<SForm.Field required>
|
||||
<label>
|
||||
Destination
|
||||
<HelpPopup content="Download any found files into this directory."/>
|
||||
</label>
|
||||
<DirectorySearch onSelect={i => this.setState({destination: i})}/>
|
||||
</SForm.Field>
|
||||
|
||||
<Grid style={{marginBottom: '0.5em'}}>
|
||||
<Grid.Row>
|
||||
<Grid.Column width={6}>
|
||||
<SForm.Field required>
|
||||
<label>
|
||||
Suffix
|
||||
<HelpPopup content='.pdf, .wav, .mp3, etc.'/>
|
||||
</label>
|
||||
<FormInput name='suffix' onChange={this.handleInputChange} placeholder='.pdf'/>
|
||||
</SForm.Field>
|
||||
</Grid.Column>
|
||||
<Grid.Column width={6}>
|
||||
<SForm.Field required>
|
||||
<label>Depth
|
||||
<HelpPopup
|
||||
content='Search the URLs provided, and any pages they contain up to this depth. Warning! This can be exponential!'/>
|
||||
</label>
|
||||
<FormDropdown selection
|
||||
name='depth'
|
||||
options={depths}
|
||||
value={depth}
|
||||
onChange={this.handleInputChange}
|
||||
/>
|
||||
</SForm.Field>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
<Grid.Row>
|
||||
<Grid.Column width={8}>
|
||||
<SForm.Field required>
|
||||
<label>
|
||||
Maximum Pages
|
||||
<HelpPopup content='Stop searching for files if this many pages have been searched.'/>
|
||||
</label>
|
||||
<FormInput
|
||||
name='max_pages'
|
||||
value={max_pages}
|
||||
onChange={this.handleInputChange}
|
||||
placeholder='100'
|
||||
/>
|
||||
</SForm.Field>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
</Grid>
|
||||
|
||||
<Button content='Cancel' onClick={this.props.clearSelected}/>
|
||||
<Button
|
||||
color='blue'
|
||||
content='Download'
|
||||
onClick={i => this.submitDownload()}
|
||||
disabled={!complete}
|
||||
/>
|
||||
</Form>
|
||||
}
|
||||
}
|
||||
|
||||
export function DownloadMenu({onOpen, disabled}) {
|
||||
const [downloader, setDownloader] = useState();
|
||||
|
||||
|
|
@ -474,6 +616,13 @@ export function DownloadMenu({onOpen, disabled}) {
|
|||
onClick={() => localOnOpen('file')}
|
||||
style={{marginBottom: '1em'}}
|
||||
/>
|
||||
<Button
|
||||
color='red'
|
||||
content='Recursive'
|
||||
disabled={disabled}
|
||||
onClick={() => localOnOpen('recursive')}
|
||||
style={{marginBottom: '1em'}}
|
||||
/>
|
||||
</>);
|
||||
|
||||
function clearSelected() {
|
||||
|
|
@ -502,6 +651,12 @@ export function DownloadMenu({onOpen, disabled}) {
|
|||
withSearchDirectory={true}
|
||||
destinationRequired={true}
|
||||
withTags={true}/>,
|
||||
recursive: <RecursiveDownload
|
||||
clearSelected={clearSelected}
|
||||
downloader='recursive'
|
||||
withSearchDirectory={true}
|
||||
destinationRequired={true}
|
||||
/>,
|
||||
};
|
||||
|
||||
body = downloader in downloaders ? downloaders[downloader] : body;
|
||||
|
|
|
|||
|
|
@ -174,7 +174,7 @@ class StoppableRow extends React.Component {
|
|||
// Open downloads (/download), or external links in an anchor.
|
||||
const link = location && !location.startsWith('/download') ?
|
||||
(text) => <Link to={location}>{text}</Link> :
|
||||
(text) => <a href={url} target='_blank' rel='noopener noreferrer'>{text}</a>;
|
||||
(text) => <a href={location || url} target='_blank' rel='noopener noreferrer'>{text}</a>;
|
||||
|
||||
let completedAtCell = last_successful_download ? isoDatetimeToElapsedPopup(last_successful_download) : null;
|
||||
let buttonCell = <TableCell/>;
|
||||
|
|
|
|||
|
|
@ -114,7 +114,7 @@ def test_get_video_for_app(test_session, simple_channel, simple_video):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_delete(test_client, test_session, test_directory, channel_factory, video_factory):
|
||||
async def test_video_delete(test_async_client, test_session, test_directory, channel_factory, video_factory):
|
||||
"""Video.delete() removes the video's files, but leave the DB record."""
|
||||
channel1, channel2 = channel_factory(), channel_factory()
|
||||
vid1 = video_factory(channel_id=channel1.id, with_video_file=True, with_caption_file=True)
|
||||
|
|
@ -132,7 +132,7 @@ async def test_video_delete(test_client, test_session, test_directory, channel_f
|
|||
assert channel1.skip_download_videos is None
|
||||
assert channel2.skip_download_videos is None
|
||||
|
||||
request, response = test_client.delete(f'/api/videos/video/{vid1.id}')
|
||||
request, response = await test_async_client.delete(f'/api/videos/video/{vid1.id}')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
# Video was added to skip list.
|
||||
|
|
@ -142,7 +142,7 @@ async def test_video_delete(test_client, test_session, test_directory, channel_f
|
|||
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
|
||||
assert vid2_video_path.is_file() and vid2_info_json_path.is_file()
|
||||
|
||||
request, response = test_client.delete(f'/api/videos/video/{vid2.id}')
|
||||
request, response = await test_async_client.delete(f'/api/videos/video/{vid2.id}')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
assert test_session.query(Video).count() == 0
|
||||
|
|
@ -150,13 +150,9 @@ async def test_video_delete(test_client, test_session, test_directory, channel_f
|
|||
assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False
|
||||
|
||||
# 3 does not exist.
|
||||
request, response = test_client.delete(f'/api/videos/video/3')
|
||||
request, response = await test_async_client.delete(f'/api/videos/video/3')
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
|
||||
# Can't parse the ids.
|
||||
request, response = test_client.delete(f'/api/videos/video/3,')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrol_mode(test_async_client, test_directory, simple_channel, simple_video, wrol_mode_fixture,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import wrolpi.files.api # noqa Import from files so the blueprint is attached.
|
||||
import wrolpi.recursive_downloader # noqa
|
||||
|
||||
BEFORE_STARTUP_FUNCTIONS = []
|
||||
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ from sqlalchemy.orm import Session
|
|||
from wrolpi import dates
|
||||
from wrolpi.dates import now, from_timestamp, seconds_to_timestamp
|
||||
from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError
|
||||
from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY
|
||||
from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS
|
||||
|
||||
logger = logging.getLogger()
|
||||
ch = logging.StreamHandler()
|
||||
|
|
@ -963,8 +963,11 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo:
|
|||
if (match := FILENAME_MATCHER.match(disposition)) and (groups := match.groups()):
|
||||
info.name = groups[0]
|
||||
else:
|
||||
# No Content-Disposition with filename, use the URL name.
|
||||
parsed = urlparse(url)
|
||||
# No Content-Disposition with filename, use the Location or URL name.
|
||||
if info.location:
|
||||
parsed = urlparse(info.location)
|
||||
else:
|
||||
parsed = urlparse(url)
|
||||
info.name = parsed.path.split('/')[-1]
|
||||
|
||||
return info
|
||||
|
|
@ -1000,7 +1003,7 @@ async def download_file(url: str, output_path: pathlib.Path = None, info: Downlo
|
|||
download_logger.info(f'Starting download of {url} with {total_size} total bytes')
|
||||
if info.accept_ranges == 'bytes' or not output_path.is_file():
|
||||
with output_path.open('ab') as fh:
|
||||
headers = dict()
|
||||
headers = DEFAULT_HTTP_HEADERS.copy()
|
||||
# Check the position of append, if it is 0 then we do not need to resume.
|
||||
position = fh.tell()
|
||||
if position:
|
||||
|
|
|
|||
|
|
@ -54,6 +54,9 @@ class FileDownloader(Downloader, ABC):
|
|||
await download_file(download.url, output_path, info)
|
||||
background_task(save_and_tag(output_path, download.settings.get('tag_names')))
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
FileGroup.from_paths(session, output_path)
|
||||
|
||||
return DownloadResult(
|
||||
success=True,
|
||||
location=f'/download/{output_path.relative_to(media_directory)}'
|
||||
|
|
|
|||
118
wrolpi/recursive_downloader.py
Normal file
118
wrolpi/recursive_downloader.py
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
import pathlib
|
||||
from http import HTTPStatus
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import bs4
|
||||
|
||||
from wrolpi.common import aiohttp_session, logger
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from wrolpi.vars import DEFAULT_HTTP_HEADERS
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
def resolve_url(parent_url: str, url: str):
|
||||
parsed_domain = urlparse(parent_url)
|
||||
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme:
|
||||
# URL is absolute.
|
||||
return url
|
||||
elif parsed.path.startswith('/'):
|
||||
return f'{parsed_domain.scheme}://{parsed_domain.netloc}{url}'
|
||||
|
||||
# Assume URL is relative to the parent URL page.
|
||||
parent_url = parent_url.rstrip('/')
|
||||
return f'{parent_url}/{url}'
|
||||
|
||||
|
||||
class RecursiveHTMLDownloader(Downloader):
|
||||
"""Recursively downloads HTML pages searching for files with a particular suffix.
|
||||
|
||||
Stops recurring when depth is reached."""
|
||||
|
||||
name = 'recursive_html'
|
||||
pretty_name = 'Recursive'
|
||||
listable = False
|
||||
|
||||
def __repr__(self):
|
||||
return '<RecursiveHTMLDownloader>'
|
||||
|
||||
@staticmethod
|
||||
async def fetch_http(url: str) -> str:
|
||||
async with aiohttp_session(timeout=60 * 5) as session:
|
||||
async with session.get(url, headers=DEFAULT_HTTP_HEADERS) as response:
|
||||
logger.debug(f'Got status={response.status} from {url}')
|
||||
if response.status == HTTPStatus.OK:
|
||||
return await response.text()
|
||||
|
||||
async def do_download(self, download: Download) -> DownloadResult:
|
||||
if download.attempts > 3:
|
||||
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
|
||||
|
||||
urls = [download.url, ]
|
||||
download_urls = list()
|
||||
|
||||
depth = download.settings.get('depth') or 1
|
||||
suffix = download.settings.get('suffix')
|
||||
max_pages = download.settings.get('max_pages') or 100
|
||||
destination = download.settings.get('destination')
|
||||
|
||||
if 0 > depth > 10:
|
||||
raise UnrecoverableDownloadError('Depth must be between 0 and 100')
|
||||
if not suffix:
|
||||
raise UnrecoverableDownloadError(f'Suffix must be defined')
|
||||
if not destination:
|
||||
raise UnrecoverableDownloadError('Destination must be defined')
|
||||
destination = pathlib.Path(destination)
|
||||
if not destination.is_dir():
|
||||
raise UnrecoverableDownloadError(f'Destination does not exist: {destination}')
|
||||
|
||||
page_count = 0
|
||||
|
||||
for i in range(depth):
|
||||
# Copy the found URLs at this depth. Accumulate all anchors (<a href>) found.
|
||||
local_urls = urls.copy()
|
||||
urls = list()
|
||||
for url in local_urls:
|
||||
content = await self.fetch_http(url)
|
||||
page_count += 1
|
||||
if not content:
|
||||
logger.error(f'Failed to download url: {url}')
|
||||
continue
|
||||
|
||||
soup = bs4.BeautifulSoup(content, 'html.parser')
|
||||
for a in soup.find_all('a'):
|
||||
try:
|
||||
href: str = a['href']
|
||||
except KeyError:
|
||||
# Not a real anchor.
|
||||
logger.debug(f'Not a real anchor: {a}')
|
||||
continue
|
||||
child_url = resolve_url(url, href)
|
||||
if child_url and child_url.endswith(suffix):
|
||||
# Found a file that the User requested.
|
||||
download_urls.append(child_url)
|
||||
else:
|
||||
urls.append(child_url)
|
||||
if page_count >= max_pages:
|
||||
logger.warning('Reached max page count.')
|
||||
break
|
||||
|
||||
if not download_urls:
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error=f'No files with {suffix} found in {page_count} pages!'
|
||||
)
|
||||
|
||||
return DownloadResult(
|
||||
success=True,
|
||||
downloads=download_urls,
|
||||
settings=download.settings,
|
||||
error='Reached max page count.' if page_count >= max_pages else None,
|
||||
location=f'/files?folders={destination}'
|
||||
)
|
||||
|
||||
|
||||
recursive_html_downloader = RecursiveHTMLDownloader()
|
||||
|
|
@ -226,7 +226,17 @@ async def post_download(_: Request, body: schema.DownloadRequest):
|
|||
|
||||
excluded_urls = [i.strip() for i in body.excluded_urls.split(',')] if body.excluded_urls else None
|
||||
destination = str(get_media_directory() / body.destination) if body.destination else None
|
||||
settings = dict(excluded_urls=excluded_urls, destination=destination, tag_names=body.tag_names)
|
||||
|
||||
if body.suffix and not body.suffix.startswith('.'):
|
||||
raise InvalidDownload(f'Suffix must start with a "."')
|
||||
|
||||
# Don't overwrite settings if restarting download.
|
||||
settings = None
|
||||
if excluded_urls or destination or body.tag_names or body.suffix or body.depth:
|
||||
settings = dict(excluded_urls=excluded_urls, destination=destination, tag_names=body.tag_names,
|
||||
suffix=body.suffix,
|
||||
depth=body.depth)
|
||||
|
||||
if body.frequency:
|
||||
download_manager.recurring_download(urls[0], body.frequency, downloader_name=body.downloader,
|
||||
sub_downloader_name=body.sub_downloader, reset_attempts=True,
|
||||
|
|
|
|||
|
|
@ -56,6 +56,9 @@ class DownloadRequest:
|
|||
excluded_urls: Optional[str] = None
|
||||
destination: Optional[str] = None
|
||||
tag_names: List[str] = field(default_factory=lambda: list())
|
||||
suffix: Optional[str] = None
|
||||
depth: Optional[int] = None
|
||||
max_pages: Optional[int] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
55
wrolpi/tags/test_recursive_downloader.py
Normal file
55
wrolpi/tags/test_recursive_downloader.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
import asyncio
|
||||
|
||||
import mock
|
||||
|
||||
from wrolpi.downloader import DownloadResult
|
||||
from wrolpi.files.downloader import FileDownloader
|
||||
from wrolpi.recursive_downloader import RecursiveHTMLDownloader
|
||||
|
||||
EXAMPLE_HTTP = '''
|
||||
<html>
|
||||
<body>
|
||||
<p><a href="https://example.com/one">ignored</a></p>
|
||||
|
||||
<p><a href="https://example.com/one.pdf">example1</a></p>
|
||||
<p><a href="/two.pdf">example2</a></p>
|
||||
<p><a href="three.pdf">example3</a></p>
|
||||
<p><a href="other.html">example4</a></p>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
|
||||
|
||||
async def test_recursive_html_downloader(test_directory, test_session, test_download_manager, assert_download_urls):
|
||||
test_download_manager.register_downloader(RecursiveHTMLDownloader())
|
||||
test_download_manager.register_downloader(FileDownloader())
|
||||
|
||||
destination = test_directory / 'destination'
|
||||
destination.mkdir()
|
||||
|
||||
settings = {
|
||||
'depth': 1,
|
||||
'suffix': '.pdf',
|
||||
'destination': str(destination),
|
||||
}
|
||||
test_download_manager.create_download('https://example.com/dir', 'recursive_html', settings=settings,
|
||||
sub_downloader_name='file')
|
||||
assert_download_urls(['https://example.com/dir', ])
|
||||
|
||||
async def fake_fetch_http(self, url: str):
|
||||
return EXAMPLE_HTTP
|
||||
|
||||
async def fake_file_do_download(*a, **kwargs):
|
||||
return DownloadResult(success=True)
|
||||
|
||||
with mock.patch('wrolpi.recursive_downloader.RecursiveHTMLDownloader.fetch_http', fake_fetch_http), \
|
||||
mock.patch('wrolpi.files.downloader.FileDownloader.do_download', fake_file_do_download):
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
await asyncio.sleep(1)
|
||||
|
||||
assert_download_urls([
|
||||
'https://example.com/dir',
|
||||
'https://example.com/one.pdf',
|
||||
'https://example.com/two.pdf',
|
||||
'https://example.com/dir/three.pdf',
|
||||
])
|
||||
|
|
@ -76,3 +76,7 @@ API_DEBUG = truthy_arg(os.environ.get('API_DEBUG', False))
|
|||
FILE_REFRESH_CHUNK_SIZE = int(os.environ.get('FILE_CHUNK_SIZE', 100))
|
||||
FILE_MAX_PDF_SIZE = int(os.environ.get('FILE_MAX_PDF_SIZE', 40_000_000))
|
||||
FILE_MAX_TEXT_SIZE = int(os.environ.get('FILE_MAX_TEXT_SIZE', 100_000))
|
||||
|
||||
DEFAULT_HTTP_HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116 Safari/537.36',
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue