wrolpi/scripts/check_dead_code.py
lrnselfreliance 8edace08bd
Python packages install from hash-locked requirements; yt-dlp still floats (#913)
* Python packages install from hash-locked requirements; yt-dlp still floats

Every pip install trusted PyPI over TLS: requirements.txt pins only top-level versions
(some with >=), the controller's are almost all >=, and dependencies were never pinned, so
a forged certificate (as the eMudhra key compromise allowed) could change code in the API,
the controller, or the archive container.  scripts/lock_requirements.sh generates a
universal, hash-locked .lock beside each requirements.txt, and
install_python_requirements.sh installs it with --require-hashes at every site.  yt-dlp is
left out of the lock and upgraded afterwards so devices still get YouTube fixes; its
dependencies are locked.  The unpinned `pip install --upgrade pip` steps are gone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* The unhashed yt-dlp install cannot replace locked packages, and nothing builds from source

Review of #913 found two holes.  The yt-dlp step ran without --no-deps, so pip's
only-if-needed strategy would still fetch an unhashed dependency whenever yt-dlp's metadata
asked for a version the lock did not install.  And srt is published only as a source
archive, whose build installed setuptools with hash checking off on every install.

The yt-dlp step now uses --no-deps (its dependencies come only from the lock), both steps
are retried as the image builds did before, and installs and locks are wheels-only.  srt
3.5.3 is vendored unmodified as wrolpi/vendor/srt.py (MIT), verified against the sdist
hash the lock pinned; webvtt-py's SRT reader rejects WROLPi's own test file, so it is not
a replacement.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-01 12:00:39 -07:00

140 lines
6.3 KiB
Python
Executable file

#!/usr/bin/env python3
"""
Fail when vulture finds dead code that is not in scripts/vulture_baseline.txt.
vulture's 60% hits are mostly dataclass/schema fields and attributes set on library objects, which
nothing in Python reads by name, so this reports only what is worth acting on: unused functions,
methods, classes, properties, imports, unreachable code, unused arguments (100% confidence), and
unused module-level variables. Class fields, attributes, and variables in tests are not reported.
The baseline is keyed by file and name, not line number, so editing a file does not invalidate it.
Usage:
pip install vulture
python3 scripts/check_dead_code.py # Exits 1 if anything new is found.
python3 scripts/check_dead_code.py --update # Rewrite the baseline with the current findings.
Delete the dead code when you can. Add it to the baseline (--update) only when vulture is wrong,
for example when the name is used dynamically.
"""
import argparse
import re
import subprocess
import sys
from collections import Counter
from pathlib import Path
PROJECT_DIR = Path(__file__).resolve().parent.parent
BASELINE = PROJECT_DIR / 'scripts/vulture_baseline.txt'
# Tests and migrations are scanned so their uses count; anything they call is not dead.
PATHS = ['wrolpi', 'modules', 'controller', 'alembic', 'main.py', 'conftest.py']
EXCLUDE = ['*/venv/*', '*/.venv/*', '*/node_modules/*', '*/vendor/*']
# These register the function somewhere vulture cannot see, so their functions are never called by name.
IGNORE_DECORATORS = [
'@*.get', '@*.post', '@*.put', '@*.patch', '@*.delete', '@*.route', '@*.websocket', '@*.signal',
'@*.listener', '@*.middleware', '@*.on_request', '@*.on_response', '@*.exception',
'@*.main_process_start', '@*.main_process_ready', '@*.reload_process_start',
'@*.before_server_start', '@*.after_server_start', '@*.before_server_stop', '@*.after_server_stop',
'@*.on_event', '@*.exception_handler', '@*.validator', '@*.field_validator', '@*.model_validator',
'@register_switch_handler', '@register_job', '@register_indexer', '@register_refresh_cleanup',
'@register_modeler', '@perpetual_signal', '@event.listens_for', '@serializer', '@pytest.fixture',
]
# Alembic reads these from every migration.
IGNORE_NAMES = ['revision', 'down_revision', 'branch_labels', 'depends_on']
# Tests request fixtures as arguments for their side effects, so unused test arguments are expected.
TEST_FILE_PATTERN = re.compile(r'(^|/)(test/|test_[^/]*\.py$|conftest\.py$)')
LINE_PATTERN = re.compile(r'^(?P<path>.+?):(?P<line>\d+): (?P<message>.+) \((?P<confidence>\d+)% confidence.*\)$')
UNUSED_PATTERN = re.compile(r"^unused (?P<kind>[a-z]+) '(?P<name>[^']+)'$")
def run_vulture() -> list[str]:
cmd = [sys.executable, '-m', 'vulture', *PATHS, '--min-confidence', '60',
'--exclude', ','.join(EXCLUDE), '--ignore-decorators', ','.join(IGNORE_DECORATORS),
'--ignore-names', ','.join(IGNORE_NAMES)]
proc = subprocess.run(cmd, cwd=PROJECT_DIR, capture_output=True, text=True)
# 0 = nothing found, 3 = dead code found; anything else is a vulture error (bad input, syntax error).
if proc.returncode not in (0, 3):
sys.stderr.write(proc.stderr or proc.stdout)
if 'No module named vulture' in proc.stderr:
sys.stderr.write('Install vulture: pip install vulture\n')
sys.exit(2)
return proc.stdout.splitlines()
def is_module_level(path: str, line: int) -> bool:
lines = (PROJECT_DIR / path).read_text().splitlines()
return line <= len(lines) and lines[line - 1][:1] not in ('', ' ', '\t')
def findings() -> dict[str, list[str]]:
"""Map each reportable finding's baseline key to the vulture lines that produced it."""
found = {}
for output in run_vulture():
if not (match := LINE_PATTERN.match(output)):
continue
path, line, message = match['path'], int(match['line']), match['message']
confidence = int(match['confidence'])
if unused := UNUSED_PATTERN.match(message):
kind, name = unused['kind'], unused['name']
if kind == 'attribute':
continue
if kind == 'variable' and TEST_FILE_PATTERN.search(path):
continue
if kind == 'variable' and confidence < 100 and not is_module_level(path, line):
continue
key = f'{path}: unused {kind} {name!r}'
else:
# Unreachable code and redundant conditions are keyed by their (unique enough) message.
key = f'{path}: {message}'
found.setdefault(key, []).append(output)
return found
def read_baseline() -> Counter:
if not BASELINE.is_file():
return Counter()
lines = (i.strip() for i in BASELINE.read_text().splitlines())
return Counter(i for i in lines if i and not i.startswith('#'))
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--update', action='store_true', help='Rewrite the baseline with the current findings.')
args = parser.parse_args()
found = findings()
current = Counter({key: len(outputs) for key, outputs in found.items()})
if args.update:
header = '# Known vulture findings, see scripts/check_dead_code.py. Regenerate with --update.\n'
BASELINE.write_text(header + ''.join(f'{key}\n' for key in sorted(current.elements())))
print(f'Wrote {sum(current.values())} findings to {BASELINE.relative_to(PROJECT_DIR)}')
return
baseline = read_baseline()
new = current - baseline
stale = baseline - current
if stale:
print('These baseline entries are no longer found; remove them with --update:')
for key in sorted(stale):
print(f' {key}')
print()
if new:
print('vulture found dead code that is not in scripts/vulture_baseline.txt:')
for key in sorted(new):
for output in found[key]:
print(f' {output}')
print('\nDelete the dead code. If vulture is wrong (the name is used dynamically, for example), run\n'
'`python3 scripts/check_dead_code.py --update` and commit the baseline.')
sys.exit(1)
print(f'No new dead code ({sum(current.values())} known findings in the baseline).')
if __name__ == '__main__':
main()