Restoring Archive docker container. Allowing readability to fail.

docker-compose.yml

@@ -33,16 +33,16 @@ services:
 #    healthcheck:
 #      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8081/api/echo' ]
 
-#  archive:
-#    build:
-#      context: .
-#      dockerfile: docker/archive/Dockerfile
-#    volumes:
-#      - './docker/archive/main.py:/app/main.py'
-#    ports:
-#      - '8083:8080'
-#    healthcheck:
-#      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8080/' ]
+  archive:
+    build:
+      context: .
+      dockerfile: docker/archive/Dockerfile
+    volumes:
+      - './docker/archive/main.py:/app/main.py'
+    ports:
+      - '8083:8080'
+    healthcheck:
+      test: [ 'CMD-SHELL', 'curl http://127.0.0.1:8080/' ]
 
   app:
     depends_on:

docker/archive/main.py

@@ -11,6 +11,8 @@ import os.path
 import pathlib
 import subprocess
 import tempfile
+import traceback
+from json import JSONDecodeError
 
 from sanic import Sanic, response
 from sanic.request import Request
@@ -97,7 +99,11 @@ async def extract_readability(path: str, url: str) -> dict:
     cmd = f'readability-extractor {path} "{url}"'
     logger.debug(f'readability cmd: {cmd}')
     stdout, stderr, return_code = await check_output(cmd)
-    output = json.loads(stdout)
+    try:
+        output = json.loads(stdout)
+    except JSONDecodeError as e:
+        stderr = stderr.decode() if stderr else None
+        raise ChildProcessError(f'Failed to extract readability.  {stderr}') from e
     logger.debug(f'done readability for {url}')
     return output
 
@@ -148,7 +154,12 @@ async def post_archive(request: Request):
         singlefile, screenshot = await asyncio.gather(call_single_file(url), take_screenshot(url))
         with tempfile.NamedTemporaryFile('wb') as fh:
             fh.write(singlefile)
-            readability = await extract_readability(fh.name, url)
+            try:
+                readability = await extract_readability(fh.name, url)
+            except Exception as e:
+                # Readability had error, but its is not required.
+                logger.error(f'Failed to get readability', exc_info=e)
+                readability = None
 
         # Compress for smaller response.
         singlefile = prepare_bytes(singlefile)
@@ -163,7 +174,8 @@ async def post_archive(request: Request):
         return response.json(ret)
     except Exception as e:
         logger.error(f'Failed to archive {url}', exc_info=e)
-        return response.json({'error': f'Failed to archive {url} error was... \n\n {e}'})
+        error = str(traceback.format_exc())
+        return response.json({'error': f'Failed to archive {url} traceback is below... \n\n {error}'})
 
 
 if __name__ == '__main__':

wrolpi/root_api.py

@@ -225,7 +225,7 @@ async def post_download(_: Request, body: schema.DownloadRequest):
     else:
         download_manager.create_downloads(urls, downloader_name=body.downloader, reset_attempts=True,
                                           sub_downloader_name=body.sub_downloader, settings=settings)
-    if download_manager.disabled:
+    if download_manager.disabled.is_set() or download_manager.stopped.is_set():
         # Downloads are disabled, warn the user.
         Events.send_downloads_disabled('Download created. But, downloads are disabled.')