mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
serve, single-file and readability-extractor install from a committed lockfile (#912)
They were installed with `npm i -g`, which pinned only the three top-level versions and resolved their 156 dependencies fresh from the registry, so a forged TLS certificate (as the eMudhra key compromise allowed) could change code WROLPi runs on every archive. npm-tools/package-lock.json pins every package with its sha512, and install_npm_tools.sh runs `npm ci` and links the commands into /usr/local/bin for every install path, upgrades, both images and the archive container. The app container's unpinned, unused global serve is removed. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
ecdad6c3e0
commit
d69f8317aa
12 changed files with 1838 additions and 32 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -118,6 +118,7 @@ test/config
|
|||
|
||||
modules/node_modules/
|
||||
docker/archive/node_modules/
|
||||
npm-tools/node_modules/
|
||||
docker/archive/package-lock.json
|
||||
|
||||
# open street map files
|
||||
|
|
|
|||
|
|
@ -79,8 +79,8 @@ git -C /opt/wrolpi reset --hard HEAD
|
|||
# so the on-device repair.sh sees this build as current and never rebuilds it.
|
||||
/opt/wrolpi/scripts/build_app.sh
|
||||
|
||||
# Install Node console commands.
|
||||
npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6
|
||||
# Install serve and the archiving tools from the committed lockfile.
|
||||
/opt/wrolpi/scripts/install_npm_tools.sh
|
||||
|
||||
# Install Deno runtime. Hash pinned to detect tampering of the upstream release artifact.
|
||||
DENO_VERSION="v2.8.3"
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ ENV PATH /app/node_modules/.bin:$PATH
|
|||
|
||||
# install and cache app dependencies
|
||||
USER root
|
||||
RUN npm install -g serve
|
||||
COPY app/public /app/public
|
||||
COPY app/src /app/src
|
||||
COPY app/package.json app/package-lock.json app/tsconfig.json /app/
|
||||
|
|
|
|||
|
|
@ -9,8 +9,10 @@ RUN apt update && apt install -y python3 python3-dev python3-venv curl unzip &&
|
|||
RUN python3 -m venv /app/venv
|
||||
RUN . /app/venv/bin/activate
|
||||
|
||||
# Install a newer singlefile and Readability.
|
||||
RUN npm install -g single-file-cli@2.0.73 readability-extractor@0.0.6
|
||||
# Install single-file and Readability from the committed lockfile. Not in /usr/src/app, which is browserless's own app.
|
||||
COPY npm-tools/package.json npm-tools/package-lock.json /opt/npm-tools/
|
||||
COPY scripts/install_npm_tools.sh /opt/install_npm_tools.sh
|
||||
RUN /opt/install_npm_tools.sh /opt/npm-tools
|
||||
|
||||
# Install Deno; single-file-cli is designed for it (under node 18 it cannot reach the browser CDP
|
||||
# on IPv6-enabled hosts and never exits). Match the version pinned in scripts/upgrade.sh.
|
||||
|
|
|
|||
1728
npm-tools/package-lock.json
generated
Normal file
1728
npm-tools/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
10
npm-tools/package.json
Normal file
10
npm-tools/package.json
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"name": "wrolpi-npm-tools",
|
||||
"private": true,
|
||||
"description": "Command-line tools WROLPi runs; installed from package-lock.json by scripts/install_npm_tools.sh.",
|
||||
"dependencies": {
|
||||
"readability-extractor": "0.0.6",
|
||||
"serve": "12.0.1",
|
||||
"single-file-cli": "2.0.73"
|
||||
}
|
||||
}
|
||||
|
|
@ -21,9 +21,6 @@ EOF
|
|||
# Create WROLPi user. This user will own the media directory, API, and App.
|
||||
useradd -md /home/wrolpi wrolpi -s "$(command -v bash)"
|
||||
|
||||
# Install Node console commands.
|
||||
npm i -g serve@12.0.1 single-file-cli@2.0.73 readability-extractor@0.0.6
|
||||
|
||||
# Install Deno runtime.
|
||||
DENO_VERSION="v2.8.3"
|
||||
DENO_SHA256="d4589cc1ffcbf1995c92a0127d932aaf832ac70cfdcc6d5b7bf38043cf303575"
|
||||
|
|
@ -38,6 +35,9 @@ deno --version
|
|||
git clone -b "${WROLPI_BRANCH:-release}" https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi
|
||||
bash /opt/wrolpi-blobs/verify_wrolpi_commit.sh /opt/wrolpi HEAD /opt/wrolpi-blobs/roland@learningselfreliance.com.gpg
|
||||
rm /opt/wrolpi-blobs/verify_wrolpi_commit.sh /opt/wrolpi-blobs/roland@learningselfreliance.com.gpg
|
||||
|
||||
# Install serve and the archiving tools from the committed lockfile.
|
||||
/opt/wrolpi/scripts/install_npm_tools.sh
|
||||
# `git config --global` needs HOME, which is empty when the build runs under
|
||||
# systemd (the release poller service has no HOME); set it to root's home.
|
||||
export HOME=/root
|
||||
|
|
|
|||
|
|
@ -20,11 +20,8 @@ apt-get install -y $(cat /opt/wrolpi/debian-live-config/config/package-lists/wro
|
|||
node -v
|
||||
npm -v
|
||||
|
||||
# Install serve, and archiving tools. Check each independently so a present
|
||||
# single-file cannot mask a missing serve (which breaks wrolpi-app).
|
||||
command -v serve >/dev/null || sudo npm i -g serve@12.0.1
|
||||
single-file --version >/dev/null 2>&1 || sudo npm i -g single-file-cli@2.0.73
|
||||
command -v readability-extractor >/dev/null || sudo npm i -g readability-extractor@0.0.6
|
||||
# Install serve and the archiving tools from the committed lockfile.
|
||||
/opt/wrolpi/scripts/install_npm_tools.sh
|
||||
|
||||
# Install Deno runtime; single-file runs under it.
|
||||
command -v deno || {
|
||||
|
|
|
|||
18
scripts/install_npm_tools.sh
Executable file
18
scripts/install_npm_tools.sh
Executable file
|
|
@ -0,0 +1,18 @@
|
|||
#!/usr/bin/env bash
|
||||
# Usage: install_npm_tools.sh [NPM_TOOLS_DIR]
|
||||
#
|
||||
# Installs the npm command-line tools WROLPi runs (serve, single-file, readability-extractor) from the committed
|
||||
# npm-tools/package-lock.json. `npm ci` checks every package against its sha512 in the lockfile, so a forged TLS
|
||||
# certificate for the registry cannot change them; `npm i -g` resolved the whole dependency tree fresh. The commands
|
||||
# are linked into /usr/local/bin, where WROLPi, help.sh and the systemd units find them.
|
||||
set -e
|
||||
|
||||
NPM_TOOLS=${1:-$(dirname "$(dirname "$(realpath "$0")")")/npm-tools}
|
||||
|
||||
npm ci --prefix "${NPM_TOOLS}" --omit=dev --no-audit --no-fund
|
||||
# Remove the copies older WROLPi versions installed globally (in /usr/bin on Raspberry Pi OS) so they cannot be found
|
||||
# instead of these.
|
||||
npm rm -g serve single-file-cli readability-extractor >/dev/null 2>&1 || :
|
||||
for command in serve single-file readability-extractor; do
|
||||
ln -sf "${NPM_TOOLS}/node_modules/.bin/${command}" "/usr/local/bin/${command}"
|
||||
done
|
||||
|
|
@ -20,11 +20,8 @@ apt-get install -y $(cat /opt/wrolpi/pi-gen/stage2/04-wrolpi/01-packages-nr)
|
|||
node -v
|
||||
npm -v
|
||||
|
||||
# Install serve, and archiving tools. Check each independently so a present
|
||||
# single-file cannot mask a missing serve (which breaks wrolpi-app).
|
||||
command -v serve >/dev/null || sudo npm i -g serve@12.0.1
|
||||
single-file --version >/dev/null 2>&1 || sudo npm i -g single-file-cli@2.0.73
|
||||
command -v readability-extractor >/dev/null || sudo npm i -g readability-extractor@0.0.6
|
||||
# Install serve and the archiving tools from the committed lockfile.
|
||||
/opt/wrolpi/scripts/install_npm_tools.sh
|
||||
|
||||
# Install Deno runtime; single-file runs under it.
|
||||
command -v deno || {
|
||||
|
|
|
|||
|
|
@ -216,20 +216,9 @@ upgrade_deno() {
|
|||
|
||||
upgrade_deno || echo "Deno upgrade failed, continuing..."
|
||||
|
||||
# Upgrade single-file to the pinned version; old versions (1.x) create compressed Archives without
|
||||
# the SingleFile comment header in the prelude, which WROLPi cannot validate.
|
||||
# readability-extractor is pinned alongside it to match the install scripts.
|
||||
upgrade_singlefile() {
|
||||
SINGLE_FILE_VERSION="2.0.73"
|
||||
if [ "$(single-file --version 2>/dev/null)" != "${SINGLE_FILE_VERSION}" ]; then
|
||||
echo "Upgrading single-file-cli to ${SINGLE_FILE_VERSION}..."
|
||||
npm i -g single-file-cli@${SINGLE_FILE_VERSION} readability-extractor@0.0.6
|
||||
else
|
||||
echo "single-file-cli ${SINGLE_FILE_VERSION} already installed"
|
||||
fi
|
||||
}
|
||||
|
||||
upgrade_singlefile || echo "single-file upgrade failed, continuing..."
|
||||
# Install serve and the archiving tools at the versions in the committed lockfile; old single-file versions (1.x)
|
||||
# create compressed Archives without the SingleFile comment header in the prelude, which WROLPi cannot validate.
|
||||
/opt/wrolpi/scripts/install_npm_tools.sh || echo "npm tools install failed, continuing..."
|
||||
|
||||
# Clear Python bytecode cache before upgrading packages.
|
||||
echo "Clearing Python bytecode cache..."
|
||||
|
|
|
|||
65
wrolpi/test/test_npm_tools.py
Normal file
65
wrolpi/test/test_npm_tools.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
"""
|
||||
Tests that the npm command-line tools WROLPi runs are installed from the committed lockfile, never resolved fresh from
|
||||
the registry.
|
||||
"""
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
NPM_TOOLS = PROJECT_DIR / 'npm-tools'
|
||||
INSTALLER = 'scripts/install_npm_tools.sh'
|
||||
TOOLS = {'serve': '12.0.1', 'single-file-cli': '2.0.73', 'readability-extractor': '0.0.6'}
|
||||
INSTALL_SITES = [
|
||||
PROJECT_DIR / 'scripts/install_debian.sh',
|
||||
PROJECT_DIR / 'scripts/install_raspberrypios.sh',
|
||||
PROJECT_DIR / 'scripts/upgrade.sh',
|
||||
PROJECT_DIR / 'pi-gen/stage2/04-wrolpi/03-run-chroot.sh',
|
||||
PROJECT_DIR / 'debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot',
|
||||
PROJECT_DIR / 'docker/archive/Dockerfile',
|
||||
]
|
||||
|
||||
|
||||
def test_tools_are_pinned():
|
||||
"""Each tool is pinned exactly in package.json, and the lockfile resolves that version."""
|
||||
package = json.loads((NPM_TOOLS / 'package.json').read_text())
|
||||
assert package['dependencies'] == TOOLS
|
||||
packages = json.loads((NPM_TOOLS / 'package-lock.json').read_text())['packages']
|
||||
for name, version in TOOLS.items():
|
||||
assert packages[f'node_modules/{name}']['version'] == version
|
||||
|
||||
|
||||
def test_every_locked_package_has_an_integrity_hash():
|
||||
"""npm ci checks each tarball against its committed sha512, so the registry's TLS is not trusted alone."""
|
||||
lock = json.loads((NPM_TOOLS / 'package-lock.json').read_text())
|
||||
assert lock['lockfileVersion'] >= 2
|
||||
packages = {k: v for k, v in lock['packages'].items() if k}
|
||||
assert len(packages) > len(TOOLS)
|
||||
for name, package in packages.items():
|
||||
assert package['integrity'].startswith('sha512-'), name
|
||||
assert package['resolved'].startswith('https://registry.npmjs.org/'), name
|
||||
|
||||
|
||||
def test_no_tool_is_installed_globally():
|
||||
"""`npm i -g` resolves the whole dependency tree fresh from the registry."""
|
||||
for directory in ('scripts', 'pi-gen', 'debian-live-config', 'docker'):
|
||||
for path in (PROJECT_DIR / directory).rglob('*'):
|
||||
if path.is_file() and 'node_modules' not in path.parts:
|
||||
text = path.read_text(errors='ignore')
|
||||
for name in TOOLS:
|
||||
for command in (f'npm i -g {name}', f'npm install -g {name}'):
|
||||
assert command not in text, path
|
||||
assert 'npm i -g serve' not in text and 'npm install -g serve' not in text, path
|
||||
|
||||
|
||||
@pytest.mark.parametrize('path', INSTALL_SITES, ids=lambda i: i.parent.name + '/' + i.name)
|
||||
def test_install_sites_use_the_lockfile(path):
|
||||
assert INSTALLER.split('/')[-1] in path.read_text(), path
|
||||
|
||||
|
||||
def test_installer_uses_npm_ci():
|
||||
text = (PROJECT_DIR / INSTALLER).read_text()
|
||||
assert 'npm ci ' in text
|
||||
for command in ('serve', 'single-file', 'readability-extractor'):
|
||||
assert command in text
|
||||
Loading…
Add table
Add a link
Reference in a new issue