From d6769c49225a751f60faf09ff1aa485489b6fc9b Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 10 Nov 2021 10:11:02 -0700
Subject: [PATCH] Compressing archive bytes before response.

---
 docker-compose.yml        |  2 ++
 docker/archive/Dockerfile |  3 ++-
 docker/archive/main.py    | 33 ++++++++++++++++++++++++++-------
 modules/archive/lib.py    | 10 +++++++++-
 4 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 65024fc5..2472a21c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -37,6 +37,8 @@ services:
     build:
       context: .
       dockerfile: docker/archive/Dockerfile
+    volumes:
+      - './docker/archive/main.py:/app/main.py'
     ports:
       - '8083:8080'
     healthcheck:
diff --git a/docker/archive/Dockerfile b/docker/archive/Dockerfile
index 32c2911f..5370355f 100644
--- a/docker/archive/Dockerfile
+++ b/docker/archive/Dockerfile
@@ -10,8 +10,9 @@ RUN python3 -m ensurepip
 RUN pip3 install --no-cache --upgrade pip setuptools
 
 # Install the Python wrapper and it's requirements.
-COPY docker/archive /app
+COPY docker/archive/requirements.txt /app/requirements.txt
 RUN pip3 install -r /app/requirements.txt
+COPY docker/archive /app
 
 # Install Readability too
 RUN npm install -g 'git+https://github.com/pirate/readability-extractor'
diff --git a/docker/archive/main.py b/docker/archive/main.py
index aae4d431..dbecb698 100644
--- a/docker/archive/main.py
+++ b/docker/archive/main.py
@@ -2,9 +2,12 @@
 """
 This file is a simple Python/Sanic wrapper around the single-file CLI command.
 """
+import asyncio
 import base64
+import gzip
 import json
 import logging
+import os.path
 import subprocess
 import tempfile
 
@@ -77,32 +80,48 @@ async def take_screenshot(url: str) -> bytes:
     with tempfile.TemporaryDirectory() as tmp_dir:
         try:
             subprocess.check_output(cmd, cwd=tmp_dir)
-        except Exception as e:
+        except Exception:
             logger.error(f'Failed to screenshot {url}', exc_info=True)
             return b''
 
+        path = f'{tmp_dir}/screenshot.png'
+        size = os.path.getsize(path)
+        logger.info(f'Successful screenshot ({size} bytes) at {path}')
         try:
-            with open(f'{tmp_dir}/screenshot.png', 'rb') as fh:
+            with open(path, 'rb') as fh:
                 png = fh.read()
-                return base64.b64encode(png)
+                return png
         except FileNotFoundError:
             return b''
 
 
+def prepare_bytes(b: bytes) -> str:
+    """
+    Compress and encode bytes for smaller response.
+    """
+    b = gzip.compress(b)
+    b = base64.b64encode(b)
+    b = b.decode()
+    return b
+
+
 @app.post('/json')
 async def post_archive(request: Request):
     url = request.json['url']
-    singlefile = await call_single_file(url)
-    screenshot = await take_screenshot(url)
+    singlefile, screenshot = await asyncio.gather(call_single_file(url), take_screenshot(url))
     with tempfile.NamedTemporaryFile('wb') as fh:
         fh.write(singlefile)
         readability = await extract_readability(fh.name, url)
 
+    # Compress for smaller response.
+    singlefile = prepare_bytes(singlefile)
+    screenshot = prepare_bytes(screenshot)
+
     ret = dict(
         url=url,
-        singlefile=singlefile.decode(),
+        singlefile=singlefile,
         readability=readability,
-        screenshot=screenshot.decode(),
+        screenshot=screenshot,
     )
     return response.json(ret)
 
diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 4c85aadd..0c6a6276 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -1,5 +1,6 @@
 import asyncio
 import base64
+import gzip
 import json
 import pathlib
 import re
@@ -100,13 +101,20 @@ def request_archive(url: str):
 
     if not singlefile:
         logger.info(f'Failed to get singlefile for {url=}')
+
     if not readability:
         logger.info(f'Failed to get readability for {url=}')
+    else:
+        # Decode and decompress.
+        readability = base64.b64decode(readability)
+        readability = gzip.decompress(readability)
+
     if not screenshot:
         logger.info(f'Failed to get screenshot for {url=}')
     else:
-        # Decode screenshot to bytes.
+        # Decode and decompress.
         screenshot = base64.b64decode(screenshot)
+        screenshot = gzip.decompress(screenshot)
 
     return singlefile, readability, screenshot