mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Remove stale Postgres references after SQLite cutover.
Drop .pgpass handling, Postgres ownership repairs, and the docker db container viewability rules. Update comments and pi-gen map docs for SQLite/PMTiles. Help submodule documents the new database layout.
This commit is contained in:
parent
698e61be34
commit
4cafb9f00e
12 changed files with 23 additions and 40 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -129,6 +129,7 @@ docker-compose.override.yml
|
|||
|
||||
# test directory is used as media directory, we don't want to commit what a user downloads.
|
||||
/test
|
||||
# Legacy local Postgres data directory from pre-SQLite docker-compose (safe to delete).
|
||||
pg_data
|
||||
|
||||
# Directories used to build images
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ export const HeadlineText = ({headline, openTag = '<u>', closeTag = '</u>'}) =>
|
|||
headline = headline.replaceAll('<b>', openTag);
|
||||
headline = headline.replaceAll('</b>', closeTag);
|
||||
headline = headline.replaceAll('\n', '<br/>');
|
||||
// Postgres inserts <b>...</b> tags which we will use.
|
||||
// FTS headlines use <b>...</b> tags which we will restyle.
|
||||
return <span {...s} dangerouslySetInnerHTML={{__html: headline}}></span>
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ except ImportError:
|
|||
CONTAINER_PREFIX = os.environ.get("COMPOSE_PROJECT_NAME", "wrolpi")
|
||||
|
||||
# Containers that should not have a "View" button (non-HTTP services)
|
||||
NON_VIEWABLE_CONTAINERS = {"db", "postgres", "postgresql", "redis", "memcached", "samba"}
|
||||
NON_VIEWABLE_CONTAINERS = {"redis", "memcached", "samba"}
|
||||
|
||||
# Containers that should use HTTPS
|
||||
HTTPS_CONTAINERS = {"web"}
|
||||
|
|
|
|||
|
|
@ -127,12 +127,12 @@ class TestGetAllContainersStatus:
|
|||
assert len(result) == 1
|
||||
assert result[0]["name"] == "api"
|
||||
|
||||
def test_db_container_not_viewable(self):
|
||||
"""Database container should not be viewable (port 5432 is not HTTP)."""
|
||||
def test_samba_container_not_viewable(self):
|
||||
"""Samba container should not be viewable (SMB is not HTTP)."""
|
||||
mock_container = mock.Mock()
|
||||
mock_container.name = f"{CONTAINER_PREFIX}-db-1"
|
||||
mock_container.name = f"{CONTAINER_PREFIX}-samba-1"
|
||||
mock_container.status = "running"
|
||||
mock_container.attrs = {"NetworkSettings": {"Ports": {"5432/tcp": [{"HostPort": "5432"}]}}}
|
||||
mock_container.attrs = {"NetworkSettings": {"Ports": {"445/tcp": [{"HostPort": "445"}]}}}
|
||||
|
||||
mock_client = mock.Mock()
|
||||
mock_client.containers.list.return_value = [mock_container]
|
||||
|
|
@ -141,7 +141,7 @@ class TestGetAllContainersStatus:
|
|||
with mock.patch("controller.lib.docker_services._get_client", return_value=mock_client):
|
||||
result = get_all_containers_status()
|
||||
assert len(result) == 1
|
||||
assert result[0]["name"] == "db"
|
||||
assert result[0]["name"] == "samba"
|
||||
assert result[0]["viewable"] is False
|
||||
|
||||
def test_api_container_is_viewable(self):
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ async def test_archives_search_headline(test_session, archive_directory, archive
|
|||
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
# Postgresql uses <b>...</b> to highlight matching words.
|
||||
# FTS headlines use <b>...</b> to highlight matching words.
|
||||
assert response.json['file_groups'][0]['d_headline'] == '<b>foo</b> baz qux qux'
|
||||
assert response.json['file_groups'][1]['d_headline'] == '<b>foo</b> bar qux'
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ class Video(ModelHelper, Base):
|
|||
Index('video_channel_id_idx', 'channel_id'),
|
||||
Index('video_source_id_idx', 'source_id'),
|
||||
Index('video_view_count_idx', 'view_count'),
|
||||
# Do not reuse ids of deleted Videos (like Postgres sequences).
|
||||
# Do not reuse ids of deleted Videos (AUTOINCREMENT keeps a sequence table).
|
||||
{'sqlite_autoincrement': True},
|
||||
)
|
||||
id = Column(Integer, primary_key=True)
|
||||
|
|
|
|||
|
|
@ -6,16 +6,8 @@ See https://github.com/RPI-Distro/pi-gen
|
|||
|
||||
## Generate 64bit Raspberry Pi WROLPi image
|
||||
|
||||
1. Convert a map PBF file to your initial map DB dump:
|
||||
|
||||
docker run --rm -v /home/user/district-of-columbia-230111.osm.pbf:/data/region.osm.pbf lrnselfreliance/osm-map-dumper > pi-gen/stage2/04-wrolpi/files/map-db-gis.dump
|
||||
|
||||
The output is already compressed (PostgreSQL custom format with `--compress 9`).
|
||||
|
||||
Verify the dump format is correct:
|
||||
|
||||
file pi-gen/stage2/04-wrolpi/files/map-db-gis.dump
|
||||
# Should show: "PostgreSQL custom database dump"
|
||||
1. (Optional) Place any default map assets under `pi-gen/stage2/04-wrolpi/files/` if the image
|
||||
should ship with preloaded map data. Maps use PMTiles files served from `/media/wrolpi/map/`.
|
||||
|
||||
2. Run the build (or, with a specific git branch)
|
||||
|
||||
|
|
|
|||
13
repair.sh
13
repair.sh
|
|
@ -35,10 +35,6 @@ find /opt/wrolpi-help/venv -name "*.pyc" -delete 2>/dev/null || :
|
|||
|
||||
# Create the WROLPi user
|
||||
grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)"
|
||||
[ -f /home/wrolpi/.pgpass ] || cat >/home/wrolpi/.pgpass <<'EOF'
|
||||
127.0.0.1:5432:wrolpi:wrolpi:wrolpi
|
||||
EOF
|
||||
chmod 0600 /home/wrolpi/.pgpass
|
||||
|
||||
# Reset any inadvertent changes to the WROLPi repo. Restore ownership in case this repair script fails.
|
||||
git config --global --add safe.directory /opt/wrolpi
|
||||
|
|
@ -167,15 +163,8 @@ fi
|
|||
|
||||
# Change owner of the media directory, ignore any errors because the drive may be fat/exfat/etc.
|
||||
chown wrolpi:wrolpi /media/wrolpi 2>/dev/null || echo "Ignoring failure to change media directory permissions."
|
||||
# Ensure the config directory is owned by wrolpi so the API can write config files.
|
||||
# Ensure the config directory is owned by wrolpi so the API can write config files and the SQLite DB.
|
||||
[ -d /media/wrolpi/config ] && chown -R wrolpi:wrolpi /media/wrolpi/config 2>/dev/null || :
|
||||
# On Portable (live-boot) systems the Postgres cluster lives under config/postgresql,
|
||||
# bind-mounted over /var/lib/postgresql — so the config chown above just swept it. Restore
|
||||
# ownership only when that bind is active; on installed systems Postgres lives on the root
|
||||
# filesystem and any config/postgresql directory is inert (dead weight or a user's backup).
|
||||
if mountpoint -q /var/lib/postgresql 2>/dev/null && id postgres >/dev/null 2>&1; then
|
||||
chown -R postgres:postgres /var/lib/postgresql /etc/postgresql 2>/dev/null || :
|
||||
fi
|
||||
|
||||
# Remove immutable flag from blob files before chown (may not exist or may not be set).
|
||||
chattr -i /opt/wrolpi-blobs/* 2>/dev/null || :
|
||||
|
|
|
|||
|
|
@ -62,10 +62,8 @@ rm -f /etc/apache2/conf-available/mod_tile.conf
|
|||
rm -f /etc/apache2/sites-available/000-default.conf
|
||||
rm -rf /etc/systemd/system/postgresql@15-map.service.d
|
||||
rm -f /var/www/html/leaflet.js /var/www/html/leaflet.css
|
||||
# Remove old .pgpass entry for the map database.
|
||||
if [ -f /home/wrolpi/.pgpass ]; then
|
||||
sed -i '/5433/d' /home/wrolpi/.pgpass
|
||||
fi
|
||||
# Remove legacy Postgres client credentials (no longer used).
|
||||
rm -f /home/wrolpi/.pgpass
|
||||
|
||||
# --- Step 5: Update wrolpi.target to remove old services ---
|
||||
|
||||
|
|
|
|||
|
|
@ -102,14 +102,17 @@ find /opt/wrolpi/venv -name "*.pyc" -delete 2>/dev/null || :
|
|||
echo "WARNING: db upgrade failed (media directory unavailable?); the API will retry at startup."
|
||||
|
||||
# WROLPi now uses SQLite (inside the media directory); stop and disable PostgreSQL to free
|
||||
# system resources. The old Postgres data is NOT deleted -- to get it back:
|
||||
# sudo systemctl enable --now postgresql
|
||||
# system resources on upgrades from older installs. The old Postgres data is NOT deleted --
|
||||
# to get it back temporarily: sudo systemctl enable --now postgresql
|
||||
# To reclaim disk later: sudo apt-get remove --purge postgresql*
|
||||
if id postgres >/dev/null 2>&1; then
|
||||
echo "Disabling PostgreSQL; WROLPi now uses SQLite. Your old Postgres data is untouched."
|
||||
# Stop the umbrella and any per-cluster instances (postgresql@15-main, etc.).
|
||||
systemctl stop 'postgresql*' 2>/dev/null || :
|
||||
systemctl disable postgresql 2>/dev/null || :
|
||||
fi
|
||||
# Remove legacy Postgres client credentials (no longer used).
|
||||
rm -f /home/wrolpi/.pgpass
|
||||
|
||||
# Upgrade WROLPi Help.
|
||||
/opt/wrolpi/scripts/install_help_service.sh || echo "Help install failed."
|
||||
|
|
|
|||
|
|
@ -1195,7 +1195,7 @@ class CollectionItem(ModelHelper, Base):
|
|||
Index('idx_collection_item_file_group_id', 'file_group_id'),
|
||||
Index('idx_collection_item_position', 'position'),
|
||||
# A FileGroup/Zim-article/URL appears at most once per collection. NULLs are distinct in
|
||||
# Postgres, so each unique constraint only applies to items of that kind.
|
||||
# SQLite unique constraints, so each constraint only applies to items of that kind.
|
||||
UniqueConstraint('collection_id', 'file_group_id', name='uq_collection_file_group'),
|
||||
UniqueConstraint('collection_id', 'zim_id', 'zim_entry', name='uq_collection_zim_entry'),
|
||||
UniqueConstraint('collection_id', 'url', name='uq_collection_url'),
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ def sanitize_filename_surrogates(path: pathlib.Path) -> pathlib.Path:
|
|||
on disk to replace them with underscores.
|
||||
|
||||
Linux filesystems store filenames as bytes. Python represents invalid UTF-8
|
||||
sequences as surrogates. These surrogates cannot be stored in PostgreSQL.
|
||||
sequences as surrogates. These surrogates cannot be stored in the database.
|
||||
|
||||
Returns the (possibly new) path.
|
||||
"""
|
||||
|
|
@ -1057,7 +1057,7 @@ def search_files(search_str: str, limit: int, offset: int, mimetypes: List[str]
|
|||
@param mimetypes: Only return files that match these mimetypes.
|
||||
@param model: Only return files that match this model.
|
||||
@param tag_names: A list of tag names.
|
||||
@param headline: Includes Postgresql headline if True.
|
||||
@param headline: Includes FTS search headlines if True.
|
||||
@param months: A list of integers representing the index of the month of the year, starting at 1.
|
||||
@param order: Used to change results from most relevant to recently viewed.
|
||||
@param url: Filter by URL using case-insensitive partial match (ILIKE).
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue