diff --git a/app/src/components/Archive.js b/app/src/components/Archive.js
index 1100fbf7..a1b4fbdd 100644
--- a/app/src/components/Archive.js
+++ b/app/src/components/Archive.js
@@ -149,9 +149,7 @@ function ArchivePage() {
if (domain) {
const domainUrl = `/archive?domain=${domain}`;
domainHeader =
{data.url ? {data.url} : 'N/A'}
+{archiveFile.url ? {archiveFile.url} : 'N/A'}
{modifiedDatetimeString}
@@ -310,7 +308,7 @@ export function ArchiveCard({file}) { - + diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index b6feaecf..4baabef1 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -277,7 +277,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { export default VideoPage; -const chapterRegex = new RegExp('^(\\(?(?:((\\d?\\d):)?(?:(\\d?\\d):(\\d\\d)))\\)?)\\s+(.*)$', 'i'); +const chapterRegex = new RegExp('^(\\(?(?:((\\d?\\d):)?(?:(\\d?\\d):(\\d\\d)))\\)?)\.?\\s+(.*)$', 'i'); function formatVideoDescription(description, setVideoTime) { // Convert timestamps to links which change the video's playback location. Change hashtags to search links. diff --git a/docker/archive/Dockerfile b/docker/archive/Dockerfile index e393fc18..09f6e83a 100644 --- a/docker/archive/Dockerfile +++ b/docker/archive/Dockerfile @@ -1,4 +1,5 @@ -FROM capsulecode/singlefile +# Keep older singlefile version so we can keep screenshots at specific ratio. +FROM capsulecode/singlefile:v0.3.32 USER root WORKDIR /app @@ -14,7 +15,7 @@ COPY docker/archive/requirements.txt /app/requirements.txt RUN pip3 install -r /app/requirements.txt COPY docker/archive /app -# Install Readability too -RUN npm install -g 'git+https://github.com/pirate/readability-extractor' +# Install a newer singlefile and Readability. +RUN npm install -g single-file-cli@1.1.16 'git+https://github.com/pirate/readability-extractor' ENTRYPOINT [ "python3", "/app/main.py"] diff --git a/docker/archive/main.py b/docker/archive/main.py index 86ad2230..9b078947 100644 --- a/docker/archive/main.py +++ b/docker/archive/main.py @@ -84,7 +84,10 @@ async def call_single_file(url) -> bytes: logger.debug(f'archive cmd: {cmd}') stdout, stderr, return_code = await check_output(cmd, always_log_stderr=True) if return_code != 0 or not stdout: - raise ValueError(f'Failed to single-file {url}') + if stderr: + for line in stderr.splitlines(): + logger.error(line) + raise RuntimeError(f'Failed to single-file {url} got the following error: {stderr}') logger.debug(f'done archiving for {url}') return stdout @@ -103,19 +106,18 @@ async def extract_readability(path: str, url: str) -> dict: output = json.loads(stdout) except JSONDecodeError as e: stderr = stderr.decode() if stderr else None - raise ChildProcessError(f'Failed to extract readability. {stderr}') from e + raise RuntimeError(f'Failed to extract readability. {stderr}') from e logger.debug(f'done readability for {url}') return output async def take_screenshot(url: str) -> bytes: - logger.info(f'Screenshot: {url}') cmd = '/usr/bin/chromium-browser' \ ' --headless' \ ' --disable-gpu' \ ' --no-sandbox' \ ' --screenshot' \ - ' --window-size=1280,720' \ + ' --window-size=1280x720' \ f' "{url}"' logger.debug(f'Screenshot cmd: {cmd}') with tempfile.TemporaryDirectory() as tmp_dir: @@ -156,6 +158,7 @@ async def post_archive(request: Request): # Use html suffix so chrome screenshot recognizes it as an HTML file. with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh: fh.write(singlefile) + fh.flush() readability = None try: readability = await extract_readability(fh.name, url) diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index e54cccf4..67993644 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -57,6 +57,8 @@ class ArchiveDownloader(Downloader, ABC): if session := Session.object_session(archive): session.commit() + logger.info(f'Successfully downloaded Archive {download.url} {archive}') + return DownloadResult(success=True, location=f'/archive/{archive.id}') @optional_session diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 6fc779f4..d4cec58f 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -320,7 +320,7 @@ def parse_article_html_metadata(html: Union[bytes, str], assume_utc: bool = True if isinstance(schema, dict) and schema.get('@context') in ('https://schema.org', 'http://schema.org'): # Found https://schema.org/ if headline := schema.get('headline'): - metadata.title = metadata.title or schema.get('headline') + metadata.title = metadata.title or headline if datePublished := schema.get('datePublished'): try: metadata.published_datetime = metadata.published_datetime or dates.strpdate(datePublished) diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index f1994ac6..e22a80ca 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -665,13 +665,13 @@ def test_archive_history(test_session, archive_factory): assert not archive5.history, 'archive5 has no history' -SINGLEFILE_1 = b''' +METADATA_EXAMPLE_1 = b''' ''' -SINGLEFILE_2 = b''' +METADATA_EXAMPLE_2 = b''' @@ -687,7 +687,7 @@ SINGLEFILE_2 = b''' ''' -SINGLEFILE_3 = b''' +METADATA_EXAMPLE_3 = b'''