mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
Fail CI when vulture finds new dead code (#889)
* Fail CI when vulture finds new dead code scripts/check_dead_code.py runs vulture and compares its findings with scripts/vulture_baseline.txt. The baseline is keyed by file and name, not line number, so unrelated edits do not invalidate it. Class fields, attributes, test arguments (side-effect fixtures), and alembic revision names are not reported, because nearly all of them are false positives. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * TEMP: add a dead function to prove the Dead Code check fails Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Revert the temporary dead function Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
4365e23327
commit
f4cd3b6f77
3 changed files with 224 additions and 0 deletions
34
.github/workflows/dead-code.yml
vendored
Normal file
34
.github/workflows/dead-code.yml
vendored
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
name: Dead Code
|
||||
|
||||
# Same triggers as backend.yml: `pull_request` is unfiltered so pull requests into feature branches
|
||||
# are checked too, not only those into master.
|
||||
on:
|
||||
push:
|
||||
branches: [master, feature/*]
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
# Fails when vulture finds dead code that is not in scripts/vulture_baseline.txt. vulture only
|
||||
# parses the source, so WROLPi's requirements are not installed.
|
||||
vulture:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
# Trixie ships 3.13.
|
||||
python-version: '3.13'
|
||||
|
||||
# Pinned: a new vulture release can report findings the baseline does not know about.
|
||||
- name: Install vulture
|
||||
run: pip install vulture==2.16
|
||||
|
||||
- name: Check for new dead code
|
||||
run: python scripts/check_dead_code.py
|
||||
140
scripts/check_dead_code.py
Executable file
140
scripts/check_dead_code.py
Executable file
|
|
@ -0,0 +1,140 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fail when vulture finds dead code that is not in scripts/vulture_baseline.txt.
|
||||
|
||||
vulture's 60% hits are mostly dataclass/schema fields and attributes set on library objects, which
|
||||
nothing in Python reads by name, so this reports only what is worth acting on: unused functions,
|
||||
methods, classes, properties, imports, unreachable code, unused arguments (100% confidence), and
|
||||
unused module-level variables. Class fields, attributes, and variables in tests are not reported.
|
||||
|
||||
The baseline is keyed by file and name, not line number, so editing a file does not invalidate it.
|
||||
|
||||
Usage:
|
||||
|
||||
pip install vulture
|
||||
python3 scripts/check_dead_code.py # Exits 1 if anything new is found.
|
||||
python3 scripts/check_dead_code.py --update # Rewrite the baseline with the current findings.
|
||||
|
||||
Delete the dead code when you can. Add it to the baseline (--update) only when vulture is wrong,
|
||||
for example when the name is used dynamically.
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_DIR = Path(__file__).resolve().parent.parent
|
||||
BASELINE = PROJECT_DIR / 'scripts/vulture_baseline.txt'
|
||||
|
||||
# Tests and migrations are scanned so their uses count; anything they call is not dead.
|
||||
PATHS = ['wrolpi', 'modules', 'controller', 'alembic', 'main.py', 'conftest.py']
|
||||
EXCLUDE = ['*/venv/*', '*/.venv/*', '*/node_modules/*']
|
||||
# These register the function somewhere vulture cannot see, so their functions are never called by name.
|
||||
IGNORE_DECORATORS = [
|
||||
'@*.get', '@*.post', '@*.put', '@*.patch', '@*.delete', '@*.route', '@*.websocket', '@*.signal',
|
||||
'@*.listener', '@*.middleware', '@*.on_request', '@*.on_response', '@*.exception',
|
||||
'@*.main_process_start', '@*.main_process_ready', '@*.reload_process_start',
|
||||
'@*.before_server_start', '@*.after_server_start', '@*.before_server_stop', '@*.after_server_stop',
|
||||
'@*.on_event', '@*.exception_handler', '@*.validator', '@*.field_validator', '@*.model_validator',
|
||||
'@register_switch_handler', '@register_job', '@register_indexer', '@register_refresh_cleanup',
|
||||
'@register_modeler', '@perpetual_signal', '@event.listens_for', '@serializer', '@pytest.fixture',
|
||||
]
|
||||
# Alembic reads these from every migration.
|
||||
IGNORE_NAMES = ['revision', 'down_revision', 'branch_labels', 'depends_on']
|
||||
# Tests request fixtures as arguments for their side effects, so unused test arguments are expected.
|
||||
TEST_FILE_PATTERN = re.compile(r'(^|/)(test/|test_[^/]*\.py$|conftest\.py$)')
|
||||
|
||||
LINE_PATTERN = re.compile(r'^(?P<path>.+?):(?P<line>\d+): (?P<message>.+) \((?P<confidence>\d+)% confidence.*\)$')
|
||||
UNUSED_PATTERN = re.compile(r"^unused (?P<kind>[a-z]+) '(?P<name>[^']+)'$")
|
||||
|
||||
|
||||
def run_vulture() -> list[str]:
|
||||
cmd = [sys.executable, '-m', 'vulture', *PATHS, '--min-confidence', '60',
|
||||
'--exclude', ','.join(EXCLUDE), '--ignore-decorators', ','.join(IGNORE_DECORATORS),
|
||||
'--ignore-names', ','.join(IGNORE_NAMES)]
|
||||
proc = subprocess.run(cmd, cwd=PROJECT_DIR, capture_output=True, text=True)
|
||||
# 0 = nothing found, 3 = dead code found; anything else is a vulture error (bad input, syntax error).
|
||||
if proc.returncode not in (0, 3):
|
||||
sys.stderr.write(proc.stderr or proc.stdout)
|
||||
if 'No module named vulture' in proc.stderr:
|
||||
sys.stderr.write('Install vulture: pip install vulture\n')
|
||||
sys.exit(2)
|
||||
return proc.stdout.splitlines()
|
||||
|
||||
|
||||
def is_module_level(path: str, line: int) -> bool:
|
||||
lines = (PROJECT_DIR / path).read_text().splitlines()
|
||||
return line <= len(lines) and lines[line - 1][:1] not in ('', ' ', '\t')
|
||||
|
||||
|
||||
def findings() -> dict[str, list[str]]:
|
||||
"""Map each reportable finding's baseline key to the vulture lines that produced it."""
|
||||
found = {}
|
||||
for output in run_vulture():
|
||||
if not (match := LINE_PATTERN.match(output)):
|
||||
continue
|
||||
path, line, message = match['path'], int(match['line']), match['message']
|
||||
confidence = int(match['confidence'])
|
||||
if unused := UNUSED_PATTERN.match(message):
|
||||
kind, name = unused['kind'], unused['name']
|
||||
if kind == 'attribute':
|
||||
continue
|
||||
if kind == 'variable' and TEST_FILE_PATTERN.search(path):
|
||||
continue
|
||||
if kind == 'variable' and confidence < 100 and not is_module_level(path, line):
|
||||
continue
|
||||
key = f'{path}: unused {kind} {name!r}'
|
||||
else:
|
||||
# Unreachable code and redundant conditions are keyed by their (unique enough) message.
|
||||
key = f'{path}: {message}'
|
||||
found.setdefault(key, []).append(output)
|
||||
return found
|
||||
|
||||
|
||||
def read_baseline() -> Counter:
|
||||
if not BASELINE.is_file():
|
||||
return Counter()
|
||||
lines = (i.strip() for i in BASELINE.read_text().splitlines())
|
||||
return Counter(i for i in lines if i and not i.startswith('#'))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('--update', action='store_true', help='Rewrite the baseline with the current findings.')
|
||||
args = parser.parse_args()
|
||||
|
||||
found = findings()
|
||||
current = Counter({key: len(outputs) for key, outputs in found.items()})
|
||||
|
||||
if args.update:
|
||||
header = '# Known vulture findings, see scripts/check_dead_code.py. Regenerate with --update.\n'
|
||||
BASELINE.write_text(header + ''.join(f'{key}\n' for key in sorted(current.elements())))
|
||||
print(f'Wrote {sum(current.values())} findings to {BASELINE.relative_to(PROJECT_DIR)}')
|
||||
return
|
||||
|
||||
baseline = read_baseline()
|
||||
new = current - baseline
|
||||
stale = baseline - current
|
||||
|
||||
if stale:
|
||||
print('These baseline entries are no longer found; remove them with --update:')
|
||||
for key in sorted(stale):
|
||||
print(f' {key}')
|
||||
print()
|
||||
|
||||
if new:
|
||||
print('vulture found dead code that is not in scripts/vulture_baseline.txt:')
|
||||
for key in sorted(new):
|
||||
for output in found[key]:
|
||||
print(f' {output}')
|
||||
print('\nDelete the dead code. If vulture is wrong (the name is used dynamically, for example), run\n'
|
||||
'`python3 scripts/check_dead_code.py --update` and commit the baseline.')
|
||||
sys.exit(1)
|
||||
|
||||
print(f'No new dead code ({sum(current.values())} known findings in the baseline).')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
50
scripts/vulture_baseline.txt
Normal file
50
scripts/vulture_baseline.txt
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
# Known vulture findings, see scripts/check_dead_code.py. Regenerate with --update.
|
||||
alembic/env.py: unused variable 'context_'
|
||||
controller/api/schemas.py: unused class 'CpuStatusResponse'
|
||||
controller/api/schemas.py: unused class 'DriveStatusResponse'
|
||||
controller/api/schemas.py: unused class 'LoadStatusResponse'
|
||||
controller/api/schemas.py: unused class 'MemoryStatusResponse'
|
||||
controller/api/schemas.py: unused class 'NetworkInterfaceResponse'
|
||||
controller/api/schemas.py: unused class 'PowerStatusResponse'
|
||||
controller/test/test_admin.py: unused function '_show_proc'
|
||||
modules/archive/api.py: unused variable 'NAME'
|
||||
modules/docs/api.py: unused variable 'NAME'
|
||||
modules/docs/models.py: unused variable 'EPUB_MIMETYPE'
|
||||
modules/inventory/api.py: unused variable 'NAME'
|
||||
modules/videos/lib.py: unused property 'quiet'
|
||||
modules/videos/lib.py: unused property 'youtube_include_dash_manifest'
|
||||
wrolpi/collections/models.py: unused method 'batch_from_config'
|
||||
wrolpi/config_version.py: unused variable 'flush_context'
|
||||
wrolpi/conftest.py: unused import 'uuid1'
|
||||
wrolpi/conftest.py: unused method 'assert_event_count'
|
||||
wrolpi/conftest.py: unused method 'assert_events_contain'
|
||||
wrolpi/conftest.py: unused method 'get_by_subject'
|
||||
wrolpi/conftest.py: unused method 'set_test_exception'
|
||||
wrolpi/dates.py: unused method 'process_result_value'
|
||||
wrolpi/dates.py: unused variable 'dialect'
|
||||
wrolpi/dates.py: unused variable 'dialect'
|
||||
wrolpi/db.py: unused variable 'clause'
|
||||
wrolpi/downloader.py: unused variable 'save_download_cache_config'
|
||||
wrolpi/files/models.py: unused class 'WrolpiKV'
|
||||
wrolpi/files/models.py: unused method 'process_result_value'
|
||||
wrolpi/files/models.py: unused variable 'dialect'
|
||||
wrolpi/files/models.py: unused variable 'dialect'
|
||||
wrolpi/files/test/test_worker_move.py: unused import 'worker_module'
|
||||
wrolpi/flags.py: unused variable 'exc_tb'
|
||||
wrolpi/flags.py: unused variable 'exc_type'
|
||||
wrolpi/flags.py: unused variable 'exc_val'
|
||||
wrolpi/jobs.py: unused method 'cancel_requested'
|
||||
wrolpi/logging_extra.py: unused class 'EmptyMessageFilter'
|
||||
wrolpi/media_path.py: unused method 'process_result_value'
|
||||
wrolpi/media_path.py: unused variable 'PATH_TYPE'
|
||||
wrolpi/media_path.py: unused variable 'dialect'
|
||||
wrolpi/media_path.py: unused variable 'dialect'
|
||||
wrolpi/tags.py: unused function 'get_name_by_id_key'
|
||||
wrolpi/test/common.py: unused class 'PytestCase'
|
||||
wrolpi/test/common.py: unused method 'assertDictContains'
|
||||
wrolpi/test/common.py: unused method 'assertGreater'
|
||||
wrolpi/test/common.py: unused method 'assertItemsTruthyOrFalsey'
|
||||
wrolpi/test/common.py: unused method 'assertLength'
|
||||
wrolpi/test/common.py: unused method 'assertLess'
|
||||
wrolpi/test/common.py: unused method 'assertRaises'
|
||||
wrolpi/vars.py: unused variable 'DATETIME_FORMAT_MS'
|
||||
Loading…
Add table
Add a link
Reference in a new issue