From 8a74140f95ed967d9b0ac0b20fc6946c23e71828 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:22:21 -0600 Subject: [PATCH] ci: a cached venv outlives the interpreter it was built for (#655) Every Python job started failing in "Install Requirements" with Error: [Errno 2] No such file or directory: '/home/circleci/project/venv/bin/python3' on branches whose only changes were under app/. The same config passed two days earlier, so this is the environment moving, not the code. The cache key names the minor version (`deps-3.13-...`) but not the patch, and the cached paths include the whole `venv`. A venv's bin/python3 points at the exact interpreter that built it, so once cimg/python:3.13 takes a patch bump the restored venv refers to an interpreter the image no longer has, and creating a venv over that directory fails instead of repairing it. Two changes, both needed: - Rebuild the venv when its interpreter cannot run. This heals the restored cache whatever left it unusable, and costs a reinstall only on the first run after a bump. - Move the cache key (-5 -> -6). CircleCI keys are immutable and `save_cache` will not overwrite one, so without this the poisoned entry is restored and discarded on every run forever and is never replaced. Not reproduced locally: macOS python recreates the symlink and exits 0, so the exact crash is CI's Linux python. The guard does not depend on the mechanism -- it removes any venv whose bin/python3 will not run, which is the state the error names. Verified that the guard heals a venv with a dangling interpreter and keeps a healthy one, so the cache still pays off. Unrelated and still open: api-tests fails on master itself, in test_format_video_filename_uses_channel_name_over_info_json, which flakes under -nauto and passes alone. Co-authored-by: Claude Opus 5 (1M context) --- .circleci/config.yml | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 394e6f2d..a187e1fa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,15 +15,22 @@ jobs: - run: sudo apt-get update - run: sudo apt-get install -y ffmpeg catdoc - restore_cache: - key: deps-3.13-{{ checksum "requirements.txt" }}-5 + key: deps-3.13-{{ checksum "requirements.txt" }}-6 - run: name: Install Requirements command: | + # A restored venv points at the exact interpreter that built it. CircleCI's + # cimg/python:3.13 gets patch bumps, and this cache key does not name the + # patch, so a venv cached before a bump has a dangling bin/python3 -- and + # `python3 -m venv` over it fails with `[Errno 2]` instead of repairing it, + # which broke every Python job. Rebuild when it cannot run: that costs a + # reinstall only on the first run after a bump. + if ! ./venv/bin/python3 --version >/dev/null 2>&1; then rm -rf venv; fi python3 -m venv venv . venv/bin/activate pip install -r requirements.txt - save_cache: - key: deps-3.13-{{ checksum "requirements.txt" }}-5 + key: deps-3.13-{{ checksum "requirements.txt" }}-6 paths: - "venv" # `-nauto` parallelizes across all available cores via pytest-xdist. @@ -49,10 +56,17 @@ jobs: steps: - checkout - restore_cache: - key: deps-3.13-{{ checksum "requirements.txt" }}-5 + key: deps-3.13-{{ checksum "requirements.txt" }}-6 - run: name: Install Requirements command: | + # A restored venv points at the exact interpreter that built it. CircleCI's + # cimg/python:3.13 gets patch bumps, and this cache key does not name the + # patch, so a venv cached before a bump has a dangling bin/python3 -- and + # `python3 -m venv` over it fails with `[Errno 2]` instead of repairing it, + # which broke every Python job. Rebuild when it cannot run: that costs a + # reinstall only on the first run after a bump. + if ! ./venv/bin/python3 --version >/dev/null 2>&1; then rm -rf venv; fi python3 -m venv venv . venv/bin/activate pip install -r requirements.txt @@ -67,10 +81,17 @@ jobs: steps: - checkout - restore_cache: - key: deps-3.13-{{ checksum "requirements.txt" }}-5 + key: deps-3.13-{{ checksum "requirements.txt" }}-6 - run: name: Install Requirements command: | + # A restored venv points at the exact interpreter that built it. CircleCI's + # cimg/python:3.13 gets patch bumps, and this cache key does not name the + # patch, so a venv cached before a bump has a dangling bin/python3 -- and + # `python3 -m venv` over it fails with `[Errno 2]` instead of repairing it, + # which broke every Python job. Rebuild when it cannot run: that costs a + # reinstall only on the first run after a bump. + if ! ./venv/bin/python3 --version >/dev/null 2>&1; then rm -rf venv; fi python3 -m venv venv . venv/bin/activate pip install -r requirements.txt @@ -89,10 +110,17 @@ jobs: steps: - checkout - restore_cache: - key: deps-3.13-{{ checksum "requirements.txt" }}-5 + key: deps-3.13-{{ checksum "requirements.txt" }}-6 - run: name: Install dependencies command: | + # A restored venv points at the exact interpreter that built it. CircleCI's + # cimg/python:3.13 gets patch bumps, and this cache key does not name the + # patch, so a venv cached before a bump has a dangling bin/python3 -- and + # `python3 -m venv` over it fails with `[Errno 2]` instead of repairing it, + # which broke every Python job. Rebuild when it cannot run: that costs a + # reinstall only on the first run after a bump. + if ! ./venv/bin/python3 --version >/dev/null 2>&1; then rm -rf venv; fi python -m venv venv . venv/bin/activate pip install -r requirements.txt