Creating RecursiveHTMLDownloader which recursively searches for files in the provided URLs.

This commit is contained in:
Roland 2023-08-30 20:19:32 -06:00
parent 1b4d4d7ad5
commit 6f43f1daa0
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
12 changed files with 377 additions and 15 deletions

View file

@ -439,7 +439,18 @@ export async function getArchive(archiveId) {
}
}
export async function postDownload(urls, downloader, frequency, sub_downloader, excludedURLs, destination, tagNames) {
export async function postDownload(
urls,
downloader,
frequency,
sub_downloader,
excludedURLs,
destination,
tagNames,
depth,
suffix,
max_pages,
) {
if (!downloader) {
toast({
type: 'error',
@ -456,6 +467,9 @@ export async function postDownload(urls, downloader, frequency, sub_downloader,
frequency: frequency || null,
excluded_urls: excludedURLs,
destination: destination || null,
depth: depth || null,
suffix: suffix || null,
max_pages: max_pages || null,
};
if (sub_downloader) {
body['sub_downloader'] = sub_downloader;

View file

@ -9,6 +9,7 @@ import {AccordionContent, AccordionTitle, Form as SForm, FormDropdown} from "sem
import {Link} from "react-router-dom";
import {TagsSelector} from "../Tags";
import {toast} from "react-semantic-toasts-2";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
const validUrl = /^(http|https):\/\/[^ "]+$/;
@ -429,6 +430,147 @@ class RSSDownload extends ChannelDownload {
}
}
class RecursiveDownload extends Downloader {
constructor(props) {
super(props);
this.state = {
depth: 1,
disabled: false,
max_pages: 100,
pending: false,
ready: false,
sub_downloader: null,
success: null,
suffix: '',
urls: '',
};
}
handleInputChange = async (event, {name, value}) => {
this.setState({[name]: value});
}
submitDownload = async () => {
let {urls, destination, depth, suffix, max_pages} = this.state;
if (urls) {
this.setState({pending: true, submitted: false});
try {
let response = await postDownload(
urls,
'recursive_html',
null,
'file',
null,
destination,
null,
depth,
suffix,
max_pages,
);
if (response.status === 204) {
this.setState({urls: '', pending: false, submitted: true});
} else {
const content = await response.json();
let error = 'Failed to create download!';
if (content && content['error']) {
error = content['error'];
}
toast({type: 'error', title: 'Error', description: error});
}
} finally {
this.setState({pending: false});
}
}
}
render() {
const {urls, depth, max_pages, suffix, destination} = this.state;
const depths = [
{key: 1, text: 1, value: 1},
{key: 2, text: 2, value: 2},
{key: 3, text: 3, value: 3},
{key: 4, text: 4, value: 4},
];
const validSuffix = suffix && suffix.startsWith('.');
const complete = urls && depth && validSuffix && max_pages && destination;
return <Form>
<Header as='h4'><Icon name='file alternate' color='red'/> Recursive File Downloader</Header>
<p>Search each of the URLs for files matching the suffix (.pdf, etc.).</p>
<TextArea required
placeholder='Enter one URL per line'
name='urls'
onChange={this.handleInputChange}
value={urls}
style={{marginBottom: '1em'}}
/>
<SForm.Field required>
<label>
Destination
<HelpPopup content="Download any found files into this directory."/>
</label>
<DirectorySearch onSelect={i => this.setState({destination: i})}/>
</SForm.Field>
<Grid style={{marginBottom: '0.5em'}}>
<Grid.Row>
<Grid.Column width={6}>
<SForm.Field required>
<label>
Suffix
<HelpPopup content='.pdf, .wav, .mp3, etc.'/>
</label>
<FormInput name='suffix' onChange={this.handleInputChange} placeholder='.pdf'/>
</SForm.Field>
</Grid.Column>
<Grid.Column width={6}>
<SForm.Field required>
<label>Depth
<HelpPopup
content='Search the URLs provided, and any pages they contain up to this depth. Warning! This can be exponential!'/>
</label>
<FormDropdown selection
name='depth'
options={depths}
value={depth}
onChange={this.handleInputChange}
/>
</SForm.Field>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column width={8}>
<SForm.Field required>
<label>
Maximum Pages
<HelpPopup content='Stop searching for files if this many pages have been searched.'/>
</label>
<FormInput
name='max_pages'
value={max_pages}
onChange={this.handleInputChange}
placeholder='100'
/>
</SForm.Field>
</Grid.Column>
</Grid.Row>
</Grid>
<Button content='Cancel' onClick={this.props.clearSelected}/>
<Button
color='blue'
content='Download'
onClick={i => this.submitDownload()}
disabled={!complete}
/>
</Form>
}
}
export function DownloadMenu({onOpen, disabled}) {
const [downloader, setDownloader] = useState();
@ -474,6 +616,13 @@ export function DownloadMenu({onOpen, disabled}) {
onClick={() => localOnOpen('file')}
style={{marginBottom: '1em'}}
/>
<Button
color='red'
content='Recursive'
disabled={disabled}
onClick={() => localOnOpen('recursive')}
style={{marginBottom: '1em'}}
/>
</>);
function clearSelected() {
@ -502,6 +651,12 @@ export function DownloadMenu({onOpen, disabled}) {
withSearchDirectory={true}
destinationRequired={true}
withTags={true}/>,
recursive: <RecursiveDownload
clearSelected={clearSelected}
downloader='recursive'
withSearchDirectory={true}
destinationRequired={true}
/>,
};
body = downloader in downloaders ? downloaders[downloader] : body;

View file

@ -174,7 +174,7 @@ class StoppableRow extends React.Component {
// Open downloads (/download), or external links in an anchor.
const link = location && !location.startsWith('/download') ?
(text) => <Link to={location}>{text}</Link> :
(text) => <a href={url} target='_blank' rel='noopener noreferrer'>{text}</a>;
(text) => <a href={location || url} target='_blank' rel='noopener noreferrer'>{text}</a>;
let completedAtCell = last_successful_download ? isoDatetimeToElapsedPopup(last_successful_download) : null;
let buttonCell = <TableCell/>;

View file

@ -114,7 +114,7 @@ def test_get_video_for_app(test_session, simple_channel, simple_video):
@pytest.mark.asyncio
async def test_video_delete(test_client, test_session, test_directory, channel_factory, video_factory):
async def test_video_delete(test_async_client, test_session, test_directory, channel_factory, video_factory):
"""Video.delete() removes the video's files, but leave the DB record."""
channel1, channel2 = channel_factory(), channel_factory()
vid1 = video_factory(channel_id=channel1.id, with_video_file=True, with_caption_file=True)
@ -132,7 +132,7 @@ async def test_video_delete(test_client, test_session, test_directory, channel_f
assert channel1.skip_download_videos is None
assert channel2.skip_download_videos is None
request, response = test_client.delete(f'/api/videos/video/{vid1.id}')
request, response = await test_async_client.delete(f'/api/videos/video/{vid1.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
# Video was added to skip list.
@ -142,7 +142,7 @@ async def test_video_delete(test_client, test_session, test_directory, channel_f
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
assert vid2_video_path.is_file() and vid2_info_json_path.is_file()
request, response = test_client.delete(f'/api/videos/video/{vid2.id}')
request, response = await test_async_client.delete(f'/api/videos/video/{vid2.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Video).count() == 0
@ -150,13 +150,9 @@ async def test_video_delete(test_client, test_session, test_directory, channel_f
assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False
# 3 does not exist.
request, response = test_client.delete(f'/api/videos/video/3')
request, response = await test_async_client.delete(f'/api/videos/video/3')
assert response.status_code == HTTPStatus.NOT_FOUND
# Can't parse the ids.
request, response = test_client.delete(f'/api/videos/video/3,')
assert response.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_wrol_mode(test_async_client, test_directory, simple_channel, simple_video, wrol_mode_fixture,

View file

@ -1,4 +1,5 @@
import wrolpi.files.api # noqa Import from files so the blueprint is attached.
import wrolpi.recursive_downloader # noqa
BEFORE_STARTUP_FUNCTIONS = []

View file

@ -37,7 +37,7 @@ from sqlalchemy.orm import Session
from wrolpi import dates
from wrolpi.dates import now, from_timestamp, seconds_to_timestamp
from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError
from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY
from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS
logger = logging.getLogger()
ch = logging.StreamHandler()
@ -963,8 +963,11 @@ async def get_download_info(url: str, timeout: int = 60) -> DownloadFileInfo:
if (match := FILENAME_MATCHER.match(disposition)) and (groups := match.groups()):
info.name = groups[0]
else:
# No Content-Disposition with filename, use the URL name.
parsed = urlparse(url)
# No Content-Disposition with filename, use the Location or URL name.
if info.location:
parsed = urlparse(info.location)
else:
parsed = urlparse(url)
info.name = parsed.path.split('/')[-1]
return info
@ -1000,7 +1003,7 @@ async def download_file(url: str, output_path: pathlib.Path = None, info: Downlo
download_logger.info(f'Starting download of {url} with {total_size} total bytes')
if info.accept_ranges == 'bytes' or not output_path.is_file():
with output_path.open('ab') as fh:
headers = dict()
headers = DEFAULT_HTTP_HEADERS.copy()
# Check the position of append, if it is 0 then we do not need to resume.
position = fh.tell()
if position:

View file

@ -54,6 +54,9 @@ class FileDownloader(Downloader, ABC):
await download_file(download.url, output_path, info)
background_task(save_and_tag(output_path, download.settings.get('tag_names')))
with get_db_session(commit=True) as session:
FileGroup.from_paths(session, output_path)
return DownloadResult(
success=True,
location=f'/download/{output_path.relative_to(media_directory)}'

View file

@ -0,0 +1,118 @@
import pathlib
from http import HTTPStatus
from urllib.parse import urlparse
import bs4
from wrolpi.common import aiohttp_session, logger
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.vars import DEFAULT_HTTP_HEADERS
logger = logger.getChild(__name__)
def resolve_url(parent_url: str, url: str):
parsed_domain = urlparse(parent_url)
parsed = urlparse(url)
if parsed.scheme:
# URL is absolute.
return url
elif parsed.path.startswith('/'):
return f'{parsed_domain.scheme}://{parsed_domain.netloc}{url}'
# Assume URL is relative to the parent URL page.
parent_url = parent_url.rstrip('/')
return f'{parent_url}/{url}'
class RecursiveHTMLDownloader(Downloader):
"""Recursively downloads HTML pages searching for files with a particular suffix.
Stops recurring when depth is reached."""
name = 'recursive_html'
pretty_name = 'Recursive'
listable = False
def __repr__(self):
return '<RecursiveHTMLDownloader>'
@staticmethod
async def fetch_http(url: str) -> str:
async with aiohttp_session(timeout=60 * 5) as session:
async with session.get(url, headers=DEFAULT_HTTP_HEADERS) as response:
logger.debug(f'Got status={response.status} from {url}')
if response.status == HTTPStatus.OK:
return await response.text()
async def do_download(self, download: Download) -> DownloadResult:
if download.attempts > 3:
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
urls = [download.url, ]
download_urls = list()
depth = download.settings.get('depth') or 1
suffix = download.settings.get('suffix')
max_pages = download.settings.get('max_pages') or 100
destination = download.settings.get('destination')
if 0 > depth > 10:
raise UnrecoverableDownloadError('Depth must be between 0 and 100')
if not suffix:
raise UnrecoverableDownloadError(f'Suffix must be defined')
if not destination:
raise UnrecoverableDownloadError('Destination must be defined')
destination = pathlib.Path(destination)
if not destination.is_dir():
raise UnrecoverableDownloadError(f'Destination does not exist: {destination}')
page_count = 0
for i in range(depth):
# Copy the found URLs at this depth. Accumulate all anchors (<a href>) found.
local_urls = urls.copy()
urls = list()
for url in local_urls:
content = await self.fetch_http(url)
page_count += 1
if not content:
logger.error(f'Failed to download url: {url}')
continue
soup = bs4.BeautifulSoup(content, 'html.parser')
for a in soup.find_all('a'):
try:
href: str = a['href']
except KeyError:
# Not a real anchor.
logger.debug(f'Not a real anchor: {a}')
continue
child_url = resolve_url(url, href)
if child_url and child_url.endswith(suffix):
# Found a file that the User requested.
download_urls.append(child_url)
else:
urls.append(child_url)
if page_count >= max_pages:
logger.warning('Reached max page count.')
break
if not download_urls:
return DownloadResult(
success=False,
error=f'No files with {suffix} found in {page_count} pages!'
)
return DownloadResult(
success=True,
downloads=download_urls,
settings=download.settings,
error='Reached max page count.' if page_count >= max_pages else None,
location=f'/files?folders={destination}'
)
recursive_html_downloader = RecursiveHTMLDownloader()

View file

@ -226,7 +226,17 @@ async def post_download(_: Request, body: schema.DownloadRequest):
excluded_urls = [i.strip() for i in body.excluded_urls.split(',')] if body.excluded_urls else None
destination = str(get_media_directory() / body.destination) if body.destination else None
settings = dict(excluded_urls=excluded_urls, destination=destination, tag_names=body.tag_names)
if body.suffix and not body.suffix.startswith('.'):
raise InvalidDownload(f'Suffix must start with a "."')
# Don't overwrite settings if restarting download.
settings = None
if excluded_urls or destination or body.tag_names or body.suffix or body.depth:
settings = dict(excluded_urls=excluded_urls, destination=destination, tag_names=body.tag_names,
suffix=body.suffix,
depth=body.depth)
if body.frequency:
download_manager.recurring_download(urls[0], body.frequency, downloader_name=body.downloader,
sub_downloader_name=body.sub_downloader, reset_attempts=True,

View file

@ -56,6 +56,9 @@ class DownloadRequest:
excluded_urls: Optional[str] = None
destination: Optional[str] = None
tag_names: List[str] = field(default_factory=lambda: list())
suffix: Optional[str] = None
depth: Optional[int] = None
max_pages: Optional[int] = None
@dataclass

View file

@ -0,0 +1,55 @@
import asyncio
import mock
from wrolpi.downloader import DownloadResult
from wrolpi.files.downloader import FileDownloader
from wrolpi.recursive_downloader import RecursiveHTMLDownloader
EXAMPLE_HTTP = '''
<html>
<body>
<p><a href="https://example.com/one">ignored</a></p>
<p><a href="https://example.com/one.pdf">example1</a></p>
<p><a href="/two.pdf">example2</a></p>
<p><a href="three.pdf">example3</a></p>
<p><a href="other.html">example4</a></p>
</body>
</html>
'''
async def test_recursive_html_downloader(test_directory, test_session, test_download_manager, assert_download_urls):
test_download_manager.register_downloader(RecursiveHTMLDownloader())
test_download_manager.register_downloader(FileDownloader())
destination = test_directory / 'destination'
destination.mkdir()
settings = {
'depth': 1,
'suffix': '.pdf',
'destination': str(destination),
}
test_download_manager.create_download('https://example.com/dir', 'recursive_html', settings=settings,
sub_downloader_name='file')
assert_download_urls(['https://example.com/dir', ])
async def fake_fetch_http(self, url: str):
return EXAMPLE_HTTP
async def fake_file_do_download(*a, **kwargs):
return DownloadResult(success=True)
with mock.patch('wrolpi.recursive_downloader.RecursiveHTMLDownloader.fetch_http', fake_fetch_http), \
mock.patch('wrolpi.files.downloader.FileDownloader.do_download', fake_file_do_download):
await test_download_manager.wait_for_all_downloads()
await asyncio.sleep(1)
assert_download_urls([
'https://example.com/dir',
'https://example.com/one.pdf',
'https://example.com/two.pdf',
'https://example.com/dir/three.pdf',
])

View file

@ -76,3 +76,7 @@ API_DEBUG = truthy_arg(os.environ.get('API_DEBUG', False))
FILE_REFRESH_CHUNK_SIZE = int(os.environ.get('FILE_CHUNK_SIZE', 100))
FILE_MAX_PDF_SIZE = int(os.environ.get('FILE_MAX_PDF_SIZE', 40_000_000))
FILE_MAX_TEXT_SIZE = int(os.environ.get('FILE_MAX_TEXT_SIZE', 100_000))
DEFAULT_HTTP_HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116 Safari/537.36',
}