Remove stale Postgres references after SQLite cutover.

Drop .pgpass handling, Postgres ownership repairs, and the docker db
container viewability rules. Update comments and pi-gen map docs for
SQLite/PMTiles. Help submodule documents the new database layout.
This commit is contained in:
Roland 2026-07-18 13:19:26 -06:00
parent 698e61be34
commit 4cafb9f00e
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
12 changed files with 23 additions and 40 deletions

1
.gitignore vendored
View file

@ -129,6 +129,7 @@ docker-compose.override.yml
# test directory is used as media directory, we don't want to commit what a user downloads.
/test
# Legacy local Postgres data directory from pre-SQLite docker-compose (safe to delete).
pg_data
# Directories used to build images

View file

@ -14,7 +14,7 @@ export const HeadlineText = ({headline, openTag = '<u>', closeTag = '</u>'}) =>
headline = headline.replaceAll('<b>', openTag);
headline = headline.replaceAll('</b>', closeTag);
headline = headline.replaceAll('\n', '<br/>');
// Postgres inserts <b>...</b> tags which we will use.
// FTS headlines use <b>...</b> tags which we will restyle.
return <span {...s} dangerouslySetInnerHTML={{__html: headline}}></span>
}
}

View file

@ -27,7 +27,7 @@ except ImportError:
CONTAINER_PREFIX = os.environ.get("COMPOSE_PROJECT_NAME", "wrolpi")
# Containers that should not have a "View" button (non-HTTP services)
NON_VIEWABLE_CONTAINERS = {"db", "postgres", "postgresql", "redis", "memcached", "samba"}
NON_VIEWABLE_CONTAINERS = {"redis", "memcached", "samba"}
# Containers that should use HTTPS
HTTPS_CONTAINERS = {"web"}

View file

@ -127,12 +127,12 @@ class TestGetAllContainersStatus:
assert len(result) == 1
assert result[0]["name"] == "api"
def test_db_container_not_viewable(self):
"""Database container should not be viewable (port 5432 is not HTTP)."""
def test_samba_container_not_viewable(self):
"""Samba container should not be viewable (SMB is not HTTP)."""
mock_container = mock.Mock()
mock_container.name = f"{CONTAINER_PREFIX}-db-1"
mock_container.name = f"{CONTAINER_PREFIX}-samba-1"
mock_container.status = "running"
mock_container.attrs = {"NetworkSettings": {"Ports": {"5432/tcp": [{"HostPort": "5432"}]}}}
mock_container.attrs = {"NetworkSettings": {"Ports": {"445/tcp": [{"HostPort": "445"}]}}}
mock_client = mock.Mock()
mock_client.containers.list.return_value = [mock_container]
@ -141,7 +141,7 @@ class TestGetAllContainersStatus:
with mock.patch("controller.lib.docker_services._get_client", return_value=mock_client):
result = get_all_containers_status()
assert len(result) == 1
assert result[0]["name"] == "db"
assert result[0]["name"] == "samba"
assert result[0]["viewable"] is False
def test_api_container_is_viewable(self):

View file

@ -136,7 +136,7 @@ async def test_archives_search_headline(test_session, archive_directory, archive
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
# Postgresql uses <b>...</b> to highlight matching words.
# FTS headlines use <b>...</b> to highlight matching words.
assert response.json['file_groups'][0]['d_headline'] == '<b>foo</b> baz qux qux'
assert response.json['file_groups'][1]['d_headline'] == '<b>foo</b> bar qux'

View file

@ -29,7 +29,7 @@ class Video(ModelHelper, Base):
Index('video_channel_id_idx', 'channel_id'),
Index('video_source_id_idx', 'source_id'),
Index('video_view_count_idx', 'view_count'),
# Do not reuse ids of deleted Videos (like Postgres sequences).
# Do not reuse ids of deleted Videos (AUTOINCREMENT keeps a sequence table).
{'sqlite_autoincrement': True},
)
id = Column(Integer, primary_key=True)

View file

@ -6,16 +6,8 @@ See https://github.com/RPI-Distro/pi-gen
## Generate 64bit Raspberry Pi WROLPi image
1. Convert a map PBF file to your initial map DB dump:
docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper > pi-gen/stage2/04-wrolpi/files/map-db-gis.dump
The output is already compressed (PostgreSQL custom format with `--compress 9`).
Verify the dump format is correct:
file pi-gen/stage2/04-wrolpi/files/map-db-gis.dump
# Should show: "PostgreSQL custom database dump"
1. (Optional) Place any default map assets under `pi-gen/stage2/04-wrolpi/files/` if the image
should ship with preloaded map data. Maps use PMTiles files served from `/media/wrolpi/map/`.
2. Run the build (or, with a specific git branch)

View file

@ -35,10 +35,6 @@ find /opt/wrolpi-help/venv -name "*.pyc" -delete 2>/dev/null || :
# Create the WROLPi user
grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)"
[ -f /home/wrolpi/.pgpass ] || cat >/home/wrolpi/.pgpass <<'EOF'
127.0.0.1:5432:wrolpi:wrolpi:wrolpi
EOF
chmod 0600 /home/wrolpi/.pgpass
# Reset any inadvertent changes to the WROLPi repo. Restore ownership in case this repair script fails.
git config --global --add safe.directory /opt/wrolpi
@ -167,15 +163,8 @@ fi
# Change owner of the media directory, ignore any errors because the drive may be fat/exfat/etc.
chown wrolpi:wrolpi /media/wrolpi 2>/dev/null || echo "Ignoring failure to change media directory permissions."
# Ensure the config directory is owned by wrolpi so the API can write config files.
# Ensure the config directory is owned by wrolpi so the API can write config files and the SQLite DB.
[ -d /media/wrolpi/config ] && chown -R wrolpi:wrolpi /media/wrolpi/config 2>/dev/null || :
# On Portable (live-boot) systems the Postgres cluster lives under config/postgresql,
# bind-mounted over /var/lib/postgresql — so the config chown above just swept it. Restore
# ownership only when that bind is active; on installed systems Postgres lives on the root
# filesystem and any config/postgresql directory is inert (dead weight or a user's backup).
if mountpoint -q /var/lib/postgresql 2>/dev/null && id postgres >/dev/null 2>&1; then
chown -R postgres:postgres /var/lib/postgresql /etc/postgresql 2>/dev/null || :
fi
# Remove immutable flag from blob files before chown (may not exist or may not be set).
chattr -i /opt/wrolpi-blobs/* 2>/dev/null || :

View file

@ -62,10 +62,8 @@ rm -f /etc/apache2/conf-available/mod_tile.conf
rm -f /etc/apache2/sites-available/000-default.conf
rm -rf /etc/systemd/system/postgresql@15-map.service.d
rm -f /var/www/html/leaflet.js /var/www/html/leaflet.css
# Remove old .pgpass entry for the map database.
if [ -f /home/wrolpi/.pgpass ]; then
sed -i '/5433/d' /home/wrolpi/.pgpass
fi
# Remove legacy Postgres client credentials (no longer used).
rm -f /home/wrolpi/.pgpass
# --- Step 5: Update wrolpi.target to remove old services ---

View file

@ -102,14 +102,17 @@ find /opt/wrolpi/venv -name "*.pyc" -delete 2>/dev/null || :
echo "WARNING: db upgrade failed (media directory unavailable?); the API will retry at startup."
# WROLPi now uses SQLite (inside the media directory); stop and disable PostgreSQL to free
# system resources. The old Postgres data is NOT deleted -- to get it back:
# sudo systemctl enable --now postgresql
# system resources on upgrades from older installs. The old Postgres data is NOT deleted --
# to get it back temporarily: sudo systemctl enable --now postgresql
# To reclaim disk later: sudo apt-get remove --purge postgresql*
if id postgres >/dev/null 2>&1; then
echo "Disabling PostgreSQL; WROLPi now uses SQLite. Your old Postgres data is untouched."
# Stop the umbrella and any per-cluster instances (postgresql@15-main, etc.).
systemctl stop 'postgresql*' 2>/dev/null || :
systemctl disable postgresql 2>/dev/null || :
fi
# Remove legacy Postgres client credentials (no longer used).
rm -f /home/wrolpi/.pgpass
# Upgrade WROLPi Help.
/opt/wrolpi/scripts/install_help_service.sh || echo "Help install failed."

View file

@ -1195,7 +1195,7 @@ class CollectionItem(ModelHelper, Base):
Index('idx_collection_item_file_group_id', 'file_group_id'),
Index('idx_collection_item_position', 'position'),
# A FileGroup/Zim-article/URL appears at most once per collection. NULLs are distinct in
# Postgres, so each unique constraint only applies to items of that kind.
# SQLite unique constraints, so each constraint only applies to items of that kind.
UniqueConstraint('collection_id', 'file_group_id', name='uq_collection_file_group'),
UniqueConstraint('collection_id', 'zim_id', 'zim_entry', name='uq_collection_zim_entry'),
UniqueConstraint('collection_id', 'url', name='uq_collection_url'),

View file

@ -74,7 +74,7 @@ def sanitize_filename_surrogates(path: pathlib.Path) -> pathlib.Path:
on disk to replace them with underscores.
Linux filesystems store filenames as bytes. Python represents invalid UTF-8
sequences as surrogates. These surrogates cannot be stored in PostgreSQL.
sequences as surrogates. These surrogates cannot be stored in the database.
Returns the (possibly new) path.
"""
@ -1057,7 +1057,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
@param mimetypes: Only return files that match these mimetypes.
@param model: Only return files that match this model.
@param tag_names: A list of tag names.
@param headline: Includes Postgresql headline if True.
@param headline: Includes FTS search headlines if True.
@param months: A list of integers representing the index of the month of the year, starting at 1.
@param order: Used to change results from most relevant to recently viewed.
@param url: Filter by URL using case-insensitive partial match (ILIKE).