diff --git a/.gitignore b/.gitignore index 00a3b3ca..a0a4d35a 100644 --- a/.gitignore +++ b/.gitignore @@ -126,3 +126,6 @@ docker-compose.override.yml # test directory is used as media directory, we don't want to commit what a user downloads. test +/debian-live-config/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz +/debian-live-config/build.log +/debian-live-config/*.iso diff --git a/.run/Development environment.run.xml b/.run/Development environment.run.xml new file mode 100644 index 00000000..e2146ec3 --- /dev/null +++ b/.run/Development environment.run.xml @@ -0,0 +1,11 @@ + + + + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index 61e62c98..7f9dc3ab 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ network so that any user with a laptop/tablet/phone can connect and use the libr * [Demo](#demo) * [Download](#download) * [Try WROLPi](#try-wrolpi) -* [Debian 11 Install](#debian-11-install) +* [Debian 12 Install](#debian-12-install) * [Raspberry Pi Install](#raspberrypi-install) * [Raspberry Pi Image Install](#raspberry-pi-image-install) * [Charter](#charter) @@ -92,13 +92,13 @@ You can try out WROLPi by running the docker containers. 9. Browse to WROLPi. * http://0.0.0.0:8080 -# Debian 11 Install +# Debian 12 Install -Install WROLPi onto a fresh Debian 11 system. +Install WROLPi onto a fresh Debian 12 system. **Warning: this will clobber anything else on the system! Do not run this installation script elsewhere!** -1. Install [Debian 11](https://www.debian.org/) onto a laptop or other computer. +1. Install [Debian 12](https://www.debian.org/) onto a laptop or other computer. 2. Get the WROLPi installation script. * `wget https://raw.githubusercontent.com/lrnselfreliance/wrolpi/release/install.sh -O /tmp/install.sh` 3. Run the installation script. diff --git a/UPGRADE.md b/UPGRADE.md index d1803fcd..a542e94f 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -3,7 +3,7 @@ # Table of Contents * [Upgrading Docker containers](#upgrading-docker-containers) -* [Upgrading Raspberry Pi or Debian 11 Installation](#upgrading-raspberry-pi-or-debian-11-installation) +* [Upgrading Raspberry Pi or Debian 12 Installation](#upgrading-raspberry-pi-or-debian-12-installation) ## Upgrading Docker containers @@ -23,7 +23,7 @@ 7. Start all docker containers * `docker-compose up -d` -## Upgrading Raspberry Pi or Debian 11 Installation +## Upgrading Raspberry Pi or Debian 12 Installation 1. Run the upgrade script * `sudo /bin/bash /opt/wrolpi/upgrade.sh` diff --git a/alembic/versions/4c8da5502c8a_drop_channel_skip_download_videos.py b/alembic/versions/4c8da5502c8a_drop_channel_skip_download_videos.py new file mode 100644 index 00000000..4e4105ee --- /dev/null +++ b/alembic/versions/4c8da5502c8a_drop_channel_skip_download_videos.py @@ -0,0 +1,31 @@ +"""Drop Channel.skip_download_videos. + +Revision ID: 4c8da5502c8a +Revises: 8bdb48a486b9 +Create Date: 2023-09-29 14:45:42.999526 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '4c8da5502c8a' +down_revision = '8bdb48a486b9' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS skip_download_videos') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE channel ADD COLUMN IF NOT EXISTS skip_download_videos TEXT[]') diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 019e113d..7c34ba4c 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -902,23 +902,50 @@ export function UnsupportedModal(header, message, icon) { export function HotspotStatusIcon() { const {on, setHotspot, dockerized} = useHotspot(); const {modal, doOpen} = UnsupportedModal('Unsupported on Docker', 'You cannot toggle the hotspot on Docker.'); + const [confirmOpen, setConfirmOpen] = React.useState(false); - const toggleHotspot = (e) => { - e.preventDefault(); - if (dockerized) { - doOpen(); - } - if (on != null) { - setHotspot(!on); + const handleConfirm = (e) => { + if (e) { + e.preventDefault() } + setConfirmOpen(false); + setHotspot(false); } - return <> - + if (dockerized === false && on === true) { + return <> + setConfirmOpen(false)} + onClose={() => setConfirmOpen(false)} + onConfirm={handleConfirm} + header='Disable the hotspot' + content='You will be disconnected when using the hotspot. Are you sure?' + confirmButton='Disable' + /> + setConfirmOpen(true)}> + + + {on === false && } + {on === null && } + + + + } else if (dockerized === false && on === false) { + return setHotspot(true)}> - - {on === false && } - {on === null && } + + + + + } + + // Hotspot is not available, or, status has not yet been fetched. + return <> + + + + {modal} @@ -929,7 +956,7 @@ export function CPUTemperatureIcon({size = 'large'}) { // Returns an Icon only if temperature is too high. const {temperature, high_temperature, critical_temperature} = useCPUTemperature(); - if (temperature < high_temperature) { + if (temperature === null || temperature < high_temperature) { // Temperature is not a problem. return null; } diff --git a/app/src/components/Download.js b/app/src/components/Download.js index da474da8..69dddb5d 100644 --- a/app/src/components/Download.js +++ b/app/src/components/Download.js @@ -33,6 +33,7 @@ class Downloader extends React.Component { submitDownload = async () => { let {urls, downloader, destination, tagNames} = this.state; if (urls) { + urls = urls.split(/\r?\n/); this.setState({pending: true, submitted: false}); try { let response = await postDownload(urls, downloader, null, null, null, destination, tagNames); @@ -195,7 +196,7 @@ class ChannelDownload extends React.Component { this.setState({error: 'URL is required'}); return; } - let response = await postDownload(url, 'video_channel', frequency, + let response = await postDownload([url], 'video_channel', frequency, null, null, destination); if (response.status === 204) { this.setState({pending: false, disabled: false, success: true, url: '', ready: false}); @@ -456,6 +457,7 @@ class ScrapeDownloader extends Downloader { depth = parseInt(depth); max_pages = parseInt(max_pages); if (urls) { + urls = urls.split(/\r?\n/); this.setState({pending: true, submitted: false}); try { let response = await postDownload( diff --git a/app/src/components/admin/Status.js b/app/src/components/admin/Status.js index 6e1d8c97..c467856b 100644 --- a/app/src/components/admin/Status.js +++ b/app/src/components/admin/Status.js @@ -143,7 +143,7 @@ export function CPUUsageProgress({percent, label}) { export function MemoryUsageProgress({percent, label}) { if (percent === null) { - return + return } let color = 'black'; @@ -176,6 +176,7 @@ export function StatusPage() { let drives = []; let disk_bandwidth = []; let memoryPercent; + let memorySize; if (status && status['cpu_info']) { const {cpu_info, load, memory_stats} = status; @@ -194,6 +195,7 @@ export function StatusPage() { disk_bandwidth = status['disk_bandwidth']; memoryPercent = Math.round(memory_stats['used'] / memory_stats['total'] * 100); + memorySize = humanFileSize(memory_stats['total'], 0); } const SizedHeader = ({children}) => { @@ -203,11 +205,14 @@ export function StatusPage() { } + const cpuProgress = ; + const memoryUsageProgress = ; + return <> - - + {cpuProgress} + {memoryUsageProgress} - - + {cpuProgress} + {memoryUsageProgress} debian-live-config/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz` +2. Run the build. + * `sudo ./build.sh` +3. The ISO is now in the `debian-live-config` directory. diff --git a/debian-live-config/build.sh b/debian-live-config/build.sh new file mode 100755 index 00000000..c6d3df3f --- /dev/null +++ b/debian-live-config/build.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +BUILD_DIR=/tmp/wrolpi-build + +# Re-execute this script if it wasn't called with sudo. +if [ $EUID != 0 ]; then + sudo "$0" "$@" + exit $? +fi + +[ -f "${SCRIPT_DIR}/config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz" ] || \ + (echo "config/includes.chroot/opt/wrolpi-blobs/gis-map.dump.gz does not exist!" && exit 1) + +# Clear out old builds. +[ -d "${BUILD_DIR}" ] && rm -rf "${BUILD_DIR}" +mkdir "${BUILD_DIR}" +cd "${BUILD_DIR}" || (echo "Work directory must exist" && exit 1) +set -e + +lb config \ + --binary-images iso-hybrid \ + --mode debian \ + --architectures amd64 \ + --linux-flavours amd64 \ + --distribution bookworm \ + --debian-installer live \ + --debian-installer-gui true \ + --archive-areas "main contrib non-free non-free-firmware" \ + --updates true \ + --security true \ + --apt-recommends false \ + --firmware-binary true \ + --firmware-chroot true \ + --win32-loader false \ + --checksums sha512 \ + --clean \ + --color \ + --linux-packages "linux-image linux-headers" \ + --memtest memtest86+ \ + --iso-preparer WROLPi \ + --iso-publisher https://wrolpi.org + +rsync -a "${SCRIPT_DIR}/config" "${BUILD_DIR}/" + +lb build 2>&1 | tee "${SCRIPT_DIR}/build.log" + +cp "${BUILD_DIR}"/live*iso "${SCRIPT_DIR}/" || (echo "Build failed. No ISOs were found!" && exit 1) +chmod 644 "${SCRIPT_DIR}"/*iso diff --git a/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot b/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot new file mode 100644 index 00000000..d0c5add1 --- /dev/null +++ b/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot @@ -0,0 +1,71 @@ +#!/usr/bin/bash +# This script is used to generate a Debian Live ISO image. + +set -e +set -x + +cat >>/etc/profile <<'EOF' +alias ll='ls -lh' +alias la='ll -a' +EOF + +git --version + +git clone -b master https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi + +# Install dependencies. +apt-get install -y < /opt/wrolpi/debian-live-config/config/package-lists/wrolpi.list.chroot + +# Install webapp. +cd /opt/wrolpi/app +npm --version +node --version +npm install +npm run build + +# Install Node console commands. +npm i -g serve@12.0.1 single-file-cli@1.0.15 readability-extractor@0.0.6 carto@1.2.0 + +# Install Python requirements. +python3 --version +python3 -m venv /opt/wrolpi/venv +/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt + +# Configure nginx. +cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf +cp /opt/wrolpi/50x.html /var/www/50x.html + +# WROLPi needs a few privileged commands. +cp /opt/wrolpi/etc/raspberrypios/90-wrolpi /etc/sudoers.d/90-wrolpi +chmod 0440 /etc/sudoers.d/90-wrolpi +# Verify this new file is valid. +visudo -c -f /etc/sudoers.d/90-wrolpi + +git clone https://github.com/lrnselfreliance/openstreetmap-carto.git /opt/openstreetmap-carto +if [[ ! -f /opt/openstreetmap-carto/mapnik.xml || ! -s /opt/openstreetmap-carto/mapnik.xml ]]; then + (cd /opt/openstreetmap-carto && carto project.mml >/opt/openstreetmap-carto/mapnik.xml) +fi + +mkdir /media/wrolpi + +cp /opt/wrolpi/etc/raspberrypios/wrolpi-api.service /etc/systemd/system/ +cp /opt/wrolpi/etc/debian12/wrolpi-app.service /etc/systemd/system/ +cp /opt/wrolpi/etc/raspberrypios/wrolpi-kiwix.service /etc/systemd/system/ +cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/ +cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf + +systemctl enable wrolpi-api.service +systemctl enable wrolpi-app.service +systemctl enable wrolpi-kiwix.service +systemctl enable renderd + +# NetworkManager for the hotspot. +systemctl enable NetworkManager + +set +x + +echo +echo "======================================================================" +echo "9999-wrolpi.hook.chroot completed" +echo "======================================================================" +echo diff --git a/debian-live-config/config/package-lists/install.list.chroot b/debian-live-config/config/package-lists/install.list.chroot new file mode 100644 index 00000000..b74eb6f7 --- /dev/null +++ b/debian-live-config/config/package-lists/install.list.chroot @@ -0,0 +1,5 @@ +git +task-gnome-desktop +task-laptop +task-ssh-server +task-xfce-desktop \ No newline at end of file diff --git a/debian-live-config/config/package-lists/wrolpi.list.chroot b/debian-live-config/config/package-lists/wrolpi.list.chroot new file mode 100644 index 00000000..a0817bca --- /dev/null +++ b/debian-live-config/config/package-lists/wrolpi.list.chroot @@ -0,0 +1,89 @@ +apache2 +apache2-dev +apt-transport-https +autoconf +build-essential +bzip2 +ca-certificates +calibre +cargo +chromium +cpufrequtils +curl +exfat-fuse +ffmpeg +firefox-esr +fonts-noto-cjk +fonts-noto-hinted +fonts-noto-unhinted +gcc +gdal-bin +git +gnupg-agent +hostapd +htop +iotop +kiwix +kiwix-tools +libagg-dev +libapache2-mod-tile +libboost-all-dev +libbz2-dev +libcairo2-dev +libcairomm-1.0-dev +libcurl4-gnutls-dev +libffi-dev +libfreetype6-dev +libgdal-dev +libgeos++-dev +libgeos-dev +libicu-dev +libiniparser-dev +liblua5.1-0-dev +liblua5.2-dev +libmapnik-dev +libmemcached-dev +libpq-dev +libproj-dev +librados-dev +libreoffice +libsensors-dev +libssl-dev +libtiff5-dev +libtool +libxml2-dev +lua5.1 +mapnik-utils +netcat-openbsd +network-manager +nginx-doc +nginx-full +nodejs +npm +osm2pgsql +osmium-tool +pkg-config +postgis +postgresql-15 +postgresql-15-postgis-3 +postgresql-15-postgis-3-scripts +postgresql-doc +protobuf-c-compiler +pv +python3-dev +python3-doc +python3-full +python3-mapnik +python3-minimal +python3-pip +python3-venv +renderd +software-properties-common +sudo +tar +tmux +unzip +vim +vlc +wget +zim-tools \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 305bd612..3a53c797 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -86,7 +86,6 @@ services: - ${WEB_HOST-0.0.0.0}:${WEB_PORT-8080}:80 volumes: - './nginx.dev.conf:/etc/nginx/nginx.conf' - - './50x.html:/var/www/50x.html' - './icon.ico:/etc/nginx/favicon.ico' - './icon.png:/etc/nginx/icon.png' - './test:/opt/media' diff --git a/etc/README.md b/etc/README.md index 7b9098f2..bfadac62 100644 --- a/etc/README.md +++ b/etc/README.md @@ -1 +1,5 @@ -The files in `etc` are files necessary to build an on-the-metal WROLPi. +# Configuration Files + +The files in `etc` are files necessary to build an on-the-metal WROLPi. Raspberry Pi has more +config files because it is assumed to be the main OS. Debian uses the Raspberry Pi configs when +possible. diff --git a/etc/debian11/renderd.service b/etc/debian11/renderd.service deleted file mode 100644 index 0dfce204..00000000 --- a/etc/debian11/renderd.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Daemon that renders map tiles using mapnik -Documentation=man:renderd -After=network.target auditd.service - -[Service] -ExecStart=/usr/bin/renderd -f -c /etc/renderd.conf -User=_renderd - -[Install] -WantedBy=multi-user.target diff --git a/etc/debian11/000-default.conf b/etc/debian12/000-default.conf similarity index 100% rename from etc/debian11/000-default.conf rename to etc/debian12/000-default.conf diff --git a/etc/debian11/wrolpi-app.service b/etc/debian12/wrolpi-app.service similarity index 79% rename from etc/debian11/wrolpi-app.service rename to etc/debian12/wrolpi-app.service index 11e59b43..4f6c5e49 100644 --- a/etc/debian11/wrolpi-app.service +++ b/etc/debian12/wrolpi-app.service @@ -4,7 +4,7 @@ Wants=wrolpi-api.service PartOf=wrolpi.target [Service] -ExecStart=/usr/bin/serve -s build -l 5000 +ExecStart=/usr/local/bin/serve -s build -l 5000 Group=wrolpi User=wrolpi WorkingDirectory=/opt/wrolpi/app/ diff --git a/50x.html b/etc/raspberrypios/50x.html similarity index 100% rename from 50x.html rename to etc/raspberrypios/50x.html diff --git a/etc/raspberrypios/90-wrolpi b/etc/raspberrypios/90-wrolpi new file mode 100644 index 00000000..3b2a36f7 --- /dev/null +++ b/etc/raspberrypios/90-wrolpi @@ -0,0 +1,15 @@ +%wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli,/usr/bin/cpufreq-set +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart renderd.service +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop renderd.service +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start renderd.service +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-kiwix.service +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-kiwix.service +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-kiwix.service +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-api.service +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-api.service +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-api.service +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-app.service +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-app.service +%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-app.service +%wrolpi ALL= NOPASSWD:/opt/wrolpi/scripts/import_map.sh +%wrolpi ALL= NOPASSWD:/usr/sbin/shutdown \ No newline at end of file diff --git a/nginx.conf b/etc/raspberrypios/nginx.conf similarity index 100% rename from nginx.conf rename to etc/raspberrypios/nginx.conf diff --git a/etc/raspberrypios/wrolpi.target b/etc/raspberrypios/wrolpi.target index bcf62ded..3a15c792 100644 --- a/etc/raspberrypios/wrolpi.target +++ b/etc/raspberrypios/wrolpi.target @@ -1,3 +1,3 @@ [Unit] Description=WROLPi services group -Wants=wrolpi-api.service wrolpi-app.service nginx.service postgresql.service +Wants=wrolpi-api.service wrolpi-app.service nginx.service postgresql.service wrolpi-kiwix.service renderd.service apache2.service diff --git a/help.sh b/help.sh index d560a552..8fc8851c 100755 --- a/help.sh +++ b/help.sh @@ -32,18 +32,18 @@ function check_file() { echo rpi=false -debian11=false +debian12=false if (grep 'Raspberry Pi' /proc/cpuinfo >/dev/null); then rpi=true fi -if (grep 'PRETTY_NAME="Debian GNU/Linux 11 (bullseye)"' /etc/os-release >/dev/null); then - debian11=true +if (grep 'PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"' /etc/os-release >/dev/null); then + debian12=true fi if [[ ${rpi} == true ]]; then echo 'OK: Running on Raspberry Pi' -elif [[ ${debian11} == true ]]; then - echo 'OK: Running on Debian 11' +elif [[ ${debian12} == true ]]; then + echo 'OK: Running on Debian 12' else echo "FAILED: Running on unknown operating system" fi @@ -169,7 +169,7 @@ else echo "FAILED: WROLPi app did not respond with interface" fi -if netstat -ant | grep LISTEN | grep 127.0.0.1:80 >/dev/null; then +if netstat -ant | grep LISTEN | grep 0.0.0.0:80 >/dev/null; then echo "OK: Port 80 is occupied" else echo "FAILED: Port 80 is not occupied" @@ -245,6 +245,12 @@ else echo "FAILED: Media directory files are not being served" fi +if [ "$(stat -c '%U' /media/wrolpi/)" == 'wrolpi' ]; then + echo "OK: Media directory is owned by wrolpi user" +else + echo "FAILED: Media directory is not owned by wrolpi user" +fi + echo # 3rd party commands if single-file -h >/dev/null 2>&1; then @@ -253,7 +259,15 @@ else echo "FAILED: Singlefile cannot be run" fi -check_file /usr/bin/readability-extractor "Readability exists" "Readability does not exist" +if [ -f /usr/bin/readability-extractor ]; then + echo "OK: readability exists" +else + if [ -f /usr/local/bin/readability-extractor ]; then + echo "OK: readability exists" + else + echo "FAILED: readability does not exist" + fi +fi if wget -h >/dev/null 2>&1; then echo "OK: wget can be run" @@ -270,7 +284,11 @@ fi if chromium-browser -h >/dev/null 2>&1; then echo "OK: Chromium can be run" else - echo "FAILED: Chromium cannot be run" + if chromium -h >/dev/null 2>&1; then + echo "OK: Chromium can be run" + else + echo "FAILED: Chromium cannot be run" + fi fi if ffmpeg -h >/dev/null 2>&1; then diff --git a/install.sh b/install.sh index fde07466..b2bc26ef 100755 --- a/install.sh +++ b/install.sh @@ -31,16 +31,16 @@ while getopts ":hb:" option; do done rpi=false -debian11=false +debian12=false if (grep 'Raspberry Pi' /proc/cpuinfo >/dev/null); then rpi=true fi -if (grep 'PRETTY_NAME="Debian GNU/Linux 11 (bullseye)"' /etc/os-release >/dev/null); then - debian11=true +if (grep 'PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"' /etc/os-release >/dev/null); then + debian12=true fi -if [[ ${rpi} == false && ${debian11} == false ]]; then - echo 'This script can only be run on a Raspberry Pi or Debian 11!' && exit 2 +if [[ ${rpi} == false && ${debian12} == false ]]; then + echo 'This script can only be run on a Raspberry Pi or Debian 12!' && exit 2 fi set -x @@ -61,8 +61,8 @@ git clone https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi || : if [ ${rpi} == true ]; then /opt/wrolpi/scripts/install_raspberrypios.sh 2>&1 | tee /opt/wrolpi/install.log install_code=${PIPESTATUS[0]} -elif [ ${debian11} == true ]; then - /opt/wrolpi/scripts/install_debian_11.sh 2>&1 | tee /opt/wrolpi/install.log +elif [ ${debian12} == true ]; then + /opt/wrolpi/scripts/install_debian_12.sh 2>&1 | tee /opt/wrolpi/install.log install_code=${PIPESTATUS[0]} fi diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 79269e97..9e265d22 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -75,6 +75,7 @@ class ChannelDownloader(Downloader, ABC): @staticmethod def is_a_playlist(info: dict): # A playlist may have an id different from its channel. + logger.debug(f'is_a_playlist {info["id"]=} {info.get("channel_id")=}') return info['id'] != info.get('channel_id') async def do_download(self, download: Download) -> DownloadResult: @@ -117,6 +118,8 @@ class ChannelDownloader(Downloader, ABC): try: if not is_a_playlist: await self.prepare_channel_for_downloads(download, channel) + else: + logger.debug('Not updating channel because this is a playlist') downloads = self.get_missing_videos(download, channel) return DownloadResult( @@ -140,6 +143,7 @@ class ChannelDownloader(Downloader, ABC): @classmethod async def prepare_channel_for_downloads(cls, download: Download, channel: Channel): """Update the Channel's video catalog. Refresh the Channel's files if necessary.""" + logger.debug(f'Preparing {channel} for downloads') channel_id = channel.id update_channel_catalog(channel, download.info_json) @@ -163,10 +167,6 @@ class ChannelDownloader(Downloader, ABC): match_regex = re.compile(channel.match_regex) downloads = [i for i in downloads if (title := i.get('title')) and match_regex.match(title)] - # Do not request videos in the skip list. - skip_download_videos = channel.skip_download_videos or [] - downloads = [i for i in downloads if i['id'] not in skip_download_videos] - # Prefer `webpage_url` before `url` for all entries. downloads = [i.get('webpage_url') or i.get('url') for i in downloads] @@ -380,10 +380,7 @@ class VideoDownloader(Downloader, ABC): source_id = info.get('id') logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not ' f'attempt to download it again.') - - with get_db_session(commit=True) as session: - c = session.query(Channel).filter_by(id=channel_id).one() - c.add_video_to_skip_list(source_id) + download.add_to_skip_list() except Exception: # Could not skip this video, it may not have a channel. logger.warning(f'Could not skip video {url}') @@ -510,6 +507,9 @@ def update_channel_catalog(channel: Channel, info: dict): info_json_path = channel.directory / f'{channel.name}.info.json' with info_json_path.open('wt') as fh: json.dump(info, fh, indent=2) + logger.debug(f'Wrote channel info json to {info_json_path}') + else: + logger.debug(f'Skipping channel info json because it does not have a directory: {channel}') logger.info(f'Finished downloading video list for {channel} found {len(entries)} videos') diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 9c4e3bba..3530b7d7 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -426,7 +426,6 @@ def import_channels_config(): # values not in the config. full_data = channel.dict() full_data.update(data) - full_data['skip_download_videos'] = list(set(data.get('skip_download_videos', {}))) channel.update(full_data) if not channel.source_id and channel.url: diff --git a/modules/videos/models.py b/modules/videos/models.py index 8a822891..e367cde7 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -101,11 +101,11 @@ class Video(ModelHelper, Base): session.delete(self) def add_to_skip_list(self): - """Add this video to it's Channel's skip list.""" - if self.channel and self.source_id: - self.channel.add_video_to_skip_list(self.source_id) + """Add this video to the DownloadManager's skip list.""" if self.url: download_manager.add_to_skip_list(self.url) + else: + logger.warning(f'{self} cannot be added to skip list because it does not have a URL') def set_viewed(self): self.viewed = now() @@ -387,7 +387,6 @@ class Channel(ModelHelper, Base): url = Column(String, unique=True) match_regex = Column(String) directory: pathlib.Path = Column(MediaPathType) - skip_download_videos = Column(ARRAY(String)) generate_posters = Column(Boolean, default=False) # generating posters may delete files, and can be slow. calculate_duration = Column(Boolean, default=True) download_frequency = Column(Integer) @@ -407,14 +406,6 @@ class Channel(ModelHelper, Base): return self.id == other.id return False - def add_video_to_skip_list(self, source_id: str): - if not source_id: - raise UnknownVideo(f'Cannot skip video with empty source id: {source_id}') - - skip_download_videos = {i for i in self.skip_download_videos or [] if i} - skip_download_videos.add(source_id) - self.skip_download_videos = skip_download_videos - def delete_with_videos(self): """Delete all Video records (but not video files) related to this Channel. Then delete the Channel.""" session = Session.object_session(self) @@ -481,7 +472,6 @@ class Channel(ModelHelper, Base): generate_posters=self.generate_posters, match_regex=self.match_regex or '', name=self.name, - skip_download_videos=self.skip_download_videos or [], source_id=self.source_id, url=self.url or None, ) diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py index 4edd3008..7ccadc87 100644 --- a/modules/videos/test/test_api.py +++ b/modules/videos/test/test_api.py @@ -159,7 +159,7 @@ def test_api_download_channel(test_session, test_client, simple_channel): def test_api_download(test_session, test_client, test_directory): """A video can be downloaded.""" - content = {'urls': 'https://example.com/video1', 'downloader': 'video', 'destination': 'dest', + content = {'urls': ['https://example.com/video1', ], 'downloader': 'video', 'destination': 'dest', 'excluded_urls': 'example.com'} request, response = test_client.post('/api/download', content=json.dumps(content)) assert response.status_code == HTTPStatus.NO_CONTENT @@ -177,7 +177,7 @@ def test_api_download_video_tags(test_session, test_client, tag_factory): tag1 = tag_factory() tag2 = tag_factory() - content = {'urls': 'https://example.com/video1', 'downloader': 'video', 'tag_names': [tag1.name, tag2.name]} + content = {'urls': ['https://example.com/video1', ], 'downloader': 'video', 'tag_names': [tag1.name, tag2.name]} request, response = test_client.post('/api/download', content=json.dumps(content)) assert response.status_code == HTTPStatus.NO_CONTENT diff --git a/modules/videos/video/test/test_api.py b/modules/videos/video/test/test_api.py index 1c04270e..4d6d7ad5 100644 --- a/modules/videos/video/test/test_api.py +++ b/modules/videos/video/test/test_api.py @@ -108,17 +108,13 @@ def test_get_video_prev_next_no_upload_date(test_session, video_factory, channel assert next_ is None -def test_get_video_for_app(test_session, simple_channel, simple_video): - vid, prev, next_ = get_video_for_app(simple_video.id) - assert vid['id'] == simple_video.id - - @pytest.mark.asyncio -async def test_video_delete(test_async_client, test_session, test_directory, channel_factory, video_factory): +async def test_video_delete(test_async_client, test_session, channel_factory, video_factory, test_download_manager): """Video.delete() removes the video's files, but leave the DB record.""" channel1, channel2 = channel_factory(), channel_factory() - vid1 = video_factory(channel_id=channel1.id, with_video_file=True, with_caption_file=True) - vid2 = video_factory(channel_id=channel2.id, with_video_file=True, with_info_json=True) + vid1 = video_factory(channel_id=channel1.id, with_video_file=True, with_caption_file=True, + with_info_json={'url': '1'}) + vid2 = video_factory(channel_id=channel2.id, with_video_file=True, with_info_json={'url': '2'}) test_session.commit() assert test_session.query(Video).count() == 2 @@ -129,14 +125,12 @@ async def test_video_delete(test_async_client, test_session, test_directory, cha # No videos have been deleted yet. assert vid1_video_path.is_file() and vid1_caption_path.is_file() assert vid2_video_path.is_file() and vid2_info_json_path.is_file() - assert channel1.skip_download_videos is None - assert channel2.skip_download_videos is None request, response = await test_async_client.delete(f'/api/videos/video/{vid1.id}') assert response.status_code == HTTPStatus.NO_CONTENT # Video was added to skip list. - assert len(channel1.skip_download_videos) == 1 + assert test_download_manager.is_skipped(vid1.url) # Video was deleted. assert test_session.query(Video).count() == 1 assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False @@ -145,6 +139,7 @@ async def test_video_delete(test_async_client, test_session, test_directory, cha request, response = await test_async_client.delete(f'/api/videos/video/{vid2.id}') assert response.status_code == HTTPStatus.NO_CONTENT + assert test_download_manager.is_skipped(vid1.url, vid2.url) assert test_session.query(Video).count() == 0 assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False @@ -155,9 +150,8 @@ async def test_video_delete(test_async_client, test_session, test_directory, cha @pytest.mark.asyncio -async def test_wrol_mode(test_async_client, test_directory, simple_channel, simple_video, wrol_mode_fixture, - test_download_manager, - tag_factory): +async def test_wrol_mode(test_async_client, simple_channel, simple_video, wrol_mode_fixture, + test_download_manager, tag_factory): """Many methods are blocked when WROL Mode is enabled.""" channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory))) tag = tag_factory() diff --git a/nginx.dev.conf b/nginx.dev.conf index aca16c16..48e4be09 100644 --- a/nginx.dev.conf +++ b/nginx.dev.conf @@ -30,10 +30,6 @@ http { alias /etc/nginx/icon.png; } - location /50x.html { - alias /var/www/50x.html; - } - location / { # React APP proxy_pass http://app:3000; diff --git a/pi-gen/stage2/04-wrolpi/01-packages b/pi-gen/stage2/04-wrolpi/01-packages index f669570a..c97797ec 100644 --- a/pi-gen/stage2/04-wrolpi/01-packages +++ b/pi-gen/stage2/04-wrolpi/01-packages @@ -63,21 +63,19 @@ postgis postgresql-13 postgresql-13-postgis-3 postgresql-13-postgis-3-scripts -postgresql-contrib protobuf-c-compiler pv python3-dev python3-doc +python3-full python3-mapnik -python3-minimal python3-pip python3-venv renderd software-properties-common tar tmux -ttf-unifont unzip vim wget -zim-tools +zim-tools \ No newline at end of file diff --git a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh index 95484b04..86bce64a 100755 --- a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh +++ b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh @@ -52,24 +52,10 @@ cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf cp /opt/wrolpi/50x.html /var/www/50x.html # WROLPi needs a few privileged commands. -cat >/etc/sudoers.d/90-wrolpi <<'EOF' -%wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli,/usr/bin/cpufreq-set -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart renderd.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop renderd.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start renderd.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-kiwix.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-kiwix.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-kiwix.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-api.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-api.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-api.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-app.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-app.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-app.service -%wrolpi ALL= NOPASSWD:/opt/wrolpi/scripts/import_map.sh -%wrolpi ALL= NOPASSWD:/usr/sbin/shutdown -EOF +cp /opt/wrolpi/etc/raspberrypios/90-wrolpi /etc/sudoers.d/90-wrolpi chmod 0440 /etc/sudoers.d/90-wrolpi +# Verify this new file is valid. +visudo -c -f /etc/sudoers.d/90-wrolpi [ -d /media/wrolpi ] || mkdir /media/wrolpi chown wrolpi:wrolpi /media/wrolpi diff --git a/repair.sh b/repair.sh new file mode 100755 index 00000000..cf820ea2 --- /dev/null +++ b/repair.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# This script will attempt to repair a WROLPi installation. It will not use internet. + +cd /opt/wrolpi || (echo "Cannot repair. /opt/wrolpi does not exist" && exit 1) + +# Re-execute this script if it wasn't called with sudo. +if [ $EUID != 0 ]; then + sudo "$0" "$@" + exit $? +fi + +rpi=false +if (grep 'Raspberry Pi' /proc/cpuinfo >/dev/null); then + rpi=true +fi + +set -e +set -x + +# Stop services if they are running. +systemctl stop wrolpi-api.service || : +systemctl stop wrolpi-app.service || : +systemctl stop wrolpi-kiwix.service || : +systemctl stop nginx || : +systemctl stop renderd || : +systemctl stop apache2 || : + +# Reset any inadvertent changes to the WROLPi repo. +git reset HEAD --hard + +# Rebuild the app after reset. +(cd app && npm run build) + +# Copy configs to system. +cp /opt/wrolpi/etc/raspberrypios/nginx.conf /etc/nginx/nginx.conf +cp /opt/wrolpi/etc/raspberrypios/50x.html /var/www/50x.html + +# WROLPi needs a few privileged commands. +cp /opt/wrolpi/etc/raspberrypios/90-wrolpi /etc/sudoers.d/90-wrolpi +chmod 0440 /etc/sudoers.d/90-wrolpi +# Verify this new file is valid. +visudo -c -f /etc/sudoers.d/90-wrolpi + +# Install the systemd services +cp /opt/wrolpi/etc/raspberrypios/wrolpi-api.service /etc/systemd/system/ +if [[ ${rpi} == true ]]; then + cp /opt/wrolpi/etc/raspberrypios/wrolpi-app.service /etc/systemd/system/ +else + cp /opt/wrolpi/etc/debian12/wrolpi-app.service /etc/systemd/system/ +fi +cp /opt/wrolpi/etc/raspberrypios/wrolpi-kiwix.service /etc/systemd/system/ +cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/ +/usr/bin/systemctl daemon-reload +systemctl enable wrolpi-api.service +systemctl enable wrolpi-app.service +systemctl enable wrolpi-kiwix.service + +cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf +# Configure Apache2 to listen on 8084. +cp /opt/wrolpi/etc/raspberrypios/ports.conf /etc/apache2/ports.conf +cp /opt/wrolpi/etc/debian12/000-default.conf /etc/apache2/sites-available/000-default.conf +# Copy Leaflet files to Apache's directory so they can be used offline. +cp /opt/wrolpi/etc/raspberrypios/index.html \ + /opt/wrolpi/modules/map/leaflet.js \ + /opt/wrolpi/modules/map/leaflet.css /var/www/html/ +chmod 644 /var/www/html/* + +systemctl enable renderd +systemctl start renderd + +# Create the WROLPi user +grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)" +[ -f /home/wrolpi/.pgpass ] || cat >/home/wrolpi/.pgpass <<'EOF' +127.0.0.1:5432:gis:_renderd:wrolpi +127.0.0.1:5432:wrolpi:wrolpi:wrolpi +EOF +chmod 0600 /home/wrolpi/.pgpass + +# WROLPi needs a few privileged commands. +cp /opt/wrolpi/etc/raspberrypios/90-wrolpi /etc/sudoers.d/90-wrolpi +chmod 0440 /etc/sudoers.d/90-wrolpi +# Verify this new file is valid. +visudo -c -f /etc/sudoers.d/90-wrolpi + +# Configure Postgresql. Do this after the API is stopped. +/opt/wrolpi/scripts/initialize_api_db.sh +# wrolpi user is superuser so they can import maps. +sudo -u postgres psql -c "alter user wrolpi with superuser" + +# Configure renderd. +if [ ! -d /opt/openstreetmap-carto ]; then + echo "/opt/openstreetmap-carto does not exist! Has install completed?" && exit 1 +fi +chown -R _renderd:_renderd /opt/openstreetmap-carto +if [[ ! -f /opt/openstreetmap-carto/mapnik.xml || ! -s /opt/openstreetmap-carto/mapnik.xml ]]; then + (cd /opt/openstreetmap-carto && carto project.mml >/opt/openstreetmap-carto/mapnik.xml) +fi +chown -R _renderd:_renderd /opt/openstreetmap-carto + +cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf +# Enable mod_tile. +cp /opt/wrolpi/etc/raspberrypios/mod_tile.conf /etc/apache2/conf-available/mod_tile.conf + +carto -v + +cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf +# Configure Apache2 to listen on 8084. +cp /opt/wrolpi/etc/raspberrypios/ports.conf /etc/apache2/ports.conf +if [[ ${rpi} == true ]]; then + cp /opt/wrolpi/etc/raspberrypios/000-default.conf /etc/apache2/sites-available/000-default.conf +else + cp /opt/wrolpi/etc/debian12/000-default.conf /etc/apache2/sites-available/000-default.conf +fi +# Copy Leaflet files to Apache's directory so they can be used offline. +cp /opt/wrolpi/etc/raspberrypios/index.html \ + /opt/wrolpi/modules/map/leaflet.js \ + /opt/wrolpi/modules/map/leaflet.css /var/www/html/ +chmod 644 /var/www/html/* +systemctl enable renderd + +# Create the media directory. This should be mounted by the maintainer. +[ -d /media/wrolpi ] || mkdir /media/wrolpi +chown wrolpi:wrolpi /media/wrolpi + +chown -R wrolpi:wrolpi /home/wrolpi /opt/wrolpi* + +systemctl start wrolpi.target diff --git a/requirements.txt b/requirements.txt index 84839a8b..ab05c0fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,4 +26,4 @@ srt==3.5.3 vininfo[cli]==1.7.0 websockets==10.1 webvtt-py==0.4.6 -yt-dlp==2023.7.6 \ No newline at end of file +yt-dlp==2023.9.24 \ No newline at end of file diff --git a/scripts/build_api_and_app.sh b/scripts/build_api_and_app.sh deleted file mode 100755 index e2693270..00000000 --- a/scripts/build_api_and_app.sh +++ /dev/null @@ -1,21 +0,0 @@ -#! /usr/bin/env bash -# This script will rebuild and install dependencies of the API and App. - -set -x -set -e - -# Install any App dependencies. -cd /opt/wrolpi/app || exit 1 -npm install || npm install || npm install || npm install # try install multiple times :( -# Build app in the background. -npm run build & - -# Install python requirements in background job. -/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt & - -wait $(jobs -p) - -# Upgrade the WROLPi database. -(cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 /opt/wrolpi/main.py db upgrade) - -systemctl start wrolpi.target diff --git a/scripts/clear_map_cache.sh b/scripts/clear_map_cache.sh new file mode 100755 index 00000000..8978cd1f --- /dev/null +++ b/scripts/clear_map_cache.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +source /opt/wrolpi/wrolpi/scripts/lib.sh + +# Re-execute this script if it wasn't called with sudo. +if [ $EUID != 0 ]; then + sudo "$0" "$@" + exit $? +fi + +yes_or_no "Are you sure you want to delete map cache files?" || exit 0 + +set -x + +# Clear map tile cache only after successful import. +[ -d /var/lib/mod_tile/ajt ] && rm -rf /var/lib/mod_tile/ajt && chown -R _renderd:_renderd /var/lib/mod_tile +[ -d /var/cache/renderd/tiles/ajt ] && rm -rf /var/cache/renderd/tiles/ajt && chown -R _renderd:_renderd /var/cache/renderd + +# Restart renderd to forget the cache. +systemctl restart renderd diff --git a/scripts/import_map.sh b/scripts/import_map.sh index 2cd3a7a8..4627e006 100755 --- a/scripts/import_map.sh +++ b/scripts/import_map.sh @@ -36,8 +36,6 @@ function cleanup() { if [ -f ${MERGED_TMP_FILE} ]; then rm ${MERGED_TMP_FILE} fi - # Renderd must be restarted to forget the cache. - systemctl restart renderd } trap cleanup EXIT @@ -107,7 +105,4 @@ else fi # Clear map tile cache only after successful import. -[ -d /var/lib/mod_tile/ajt ] && rm -rf /var/lib/mod_tile/ajt -[ -d /var/cache/renderd/tiles/ajt ] && rm -rf /var/cache/renderd/tiles/ajt - -# Renderd is restarted in cleanup trap. +yes | /bin/bash /opt/wrolpi/scripts/clear_map_cache.sh diff --git a/scripts/initialize_api_db.sh b/scripts/initialize_api_db.sh new file mode 100755 index 00000000..dddc2e31 --- /dev/null +++ b/scripts/initialize_api_db.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Creates the wrolpi database user and database if they don't exist. +# It is always safe to run this script. + +# Re-execute this script if it wasn't called with sudo. +if [ $EUID != 0 ]; then + sudo "$0" "$@" + exit $? +fi + +set -x + +# Create WROLPi API database. +sudo -u postgres psql -c '\l' | grep wrolpi || ( + sudo -u postgres createuser -s wrolpi # superuser so maps can be imported + sudo -u postgres psql -c "alter user postgres password 'wrolpi'" + sudo -u postgres psql -c "alter user wrolpi password 'wrolpi'" + sudo -u postgres createdb -E UTF8 -O wrolpi wrolpi + echo "Created wrolpi database" +) +sudo -u wrolpi /bin/bash -c 'cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 main.py db upgrade' diff --git a/scripts/initialize_map.sh b/scripts/initialize_map.sh deleted file mode 100755 index c2e0db6b..00000000 --- a/scripts/initialize_map.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -if [ ! -d /opt/openstreetmap-carto ]; then - echo "/opt/openstreetmap-carto does not exist. Run install.sh!" - exit 1 -fi - -set -x - -# Use D.C. to initialized DB because it is so small. -DC_MAP=district-of-columbia-230222.osm.pbf -DC_MAP_PATH=/tmp/${DC_MAP} -[ -f "${DC_MAP_PATH}" ] || wget --continue https://wrolpi.org/downloads/${DC_MAP} -O ${DC_MAP_PATH} - -if [ ! -f "${DC_MAP_PATH}" ]; then - echo "Could not download D.C. map" - exit 2 -fi - -cd /opt/openstreetmap-carto || exit 3 - -# Initialize indexes and global polygons. This may take multiple tries. -MAX_TRIES=3 -COUNT=0 -SUCCESS=false -external_data=false -dc_import=false -indexes=false -while [ "${COUNT}" -lt "${MAX_TRIES}" ]; do - [[ "${external_data}" = false ]] && - sudo -u _renderd nice -n 18 /opt/openstreetmap-carto/scripts/get-external-data.py \ - -C \ - -d gis -U _renderd && - external_data=true - [[ "${dc_import}" = false ]] && - sudo -u _renderd nice -n 18 /opt/wrolpi/scripts/import_map.sh ${DC_MAP_PATH} && - dc_import=true - [[ "${indexes}" = false ]] && - sudo -u _renderd psql -d gis -f /opt/openstreetmap-carto/indexes.sql && - indexes=true - if [[ "${external_data}" == true && "${dc_import}" == true && "${indexes}" == true ]]; then - SUCCESS=true - break - fi - ((COUNT = COUNT + 1)) -done - -if [[ "${SUCCESS}" == false ]]; then - echo "Could not initialize map data. Try running reset_map.sh, or install.sh." - exit 4 -fi - -echo "Map has successfully been initialized" diff --git a/scripts/create_db.sh b/scripts/initialize_map_db.sh similarity index 59% rename from scripts/create_db.sh rename to scripts/initialize_map_db.sh index 75f84956..96e2c89a 100755 --- a/scripts/create_db.sh +++ b/scripts/initialize_map_db.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash +# Creates the gis (map) database and _renderd database user if they don't exist. +# WARNING: This will overwrite the map database data. -# Create WROLPi API database. -sudo -u postgres psql -c '\l' | grep wrolpi || ( - sudo -u postgres createuser -s wrolpi # superuser so maps can be imported - sudo -u postgres psql -c "alter user postgres password 'wrolpi'" - sudo -u postgres psql -c "alter user wrolpi password 'wrolpi'" - sudo -u postgres createdb -E UTF8 -O wrolpi wrolpi - echo "Created wrolpi database" -) -sudo -u wrolpi /bin/bash -c 'cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 main.py db upgrade' +source /opt/wrolpi/wrolpi/scripts/lib.sh + +yes_or_no "Are you sure you want to initialize the map database? Data from imported maps will be lost." || exit 0 + +set -x + +systemctl stop renderd # Create gis (map) database. sudo -u postgres psql -c '\l' | grep gis || ( @@ -24,5 +24,6 @@ zcat /opt/wrolpi-blobs/gis-map.dump.gz | sudo -u postgres pg_restore --no-owner sudo -u postgres psql -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd' sudo -u postgres psql -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd' sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO _renderd" + # Clear map tile cache files. -[ -d /var/cache/renderd/tiles/ajt ] && sudo rm -r /var/cache/renderd/tiles/ajt +yes | /bin/bash /opt/wrolpi/scripts/clear_map_cache.sh diff --git a/scripts/install_debian_11.sh b/scripts/install_debian_11.sh deleted file mode 100755 index 216be8a1..00000000 --- a/scripts/install_debian_11.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env bash -echo "Debian 11 install start: $(date '+%Y-%m-%d %H:%M:%S')" - -set -x -set -e - -# Update if we haven't updated in the last day. -[ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mtime -1)" ] && apt update -# Install dependencies. -apt-get install -y \ - apt-transport-https \ - ca-certificates \ - calibre \ - chromium \ - chromium-driver \ - cpufrequtils \ - curl \ - ffmpeg \ - gcc \ - gnupg-agent \ - hostapd \ - htop \ - iotop \ - kiwix \ - kiwix-tools \ - libpq-dev \ - netcat \ - network-manager \ - nginx-doc \ - nginx-full \ - postgresql-13 \ - python3-dev \ - python3-doc \ - python3-full \ - python3-pip \ - python3-venv \ - software-properties-common \ - tmux \ - vim \ - zim-tools - -# Install npm. -npm --version || curl -fsSL https://deb.nodesource.com/setup_18.x | bash - -apt-get install -y nodejs -node -v -npm -v - -# Install serve, and archiving tools. -sudo npm i -g serve@12.0.1 single-file-cli@1.0.33 readability-extractor@0.0.6 - -# Build React app in background job. -cd /opt/wrolpi/app || exit 5 -npm install || npm install || npm install || npm install # try install multiple times :( -npm run build & - -# Install python requirements in background job. -python3 -m venv /opt/wrolpi/venv -/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt & - -wait $(jobs -p) - -# Install the WROLPi nginx config over the default nginx config. -cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf -cp /opt/wrolpi/50x.html /var/www/50x.html -/usr/sbin/nginx -s reload - -# Create the WROLPi user -grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)" -cat >/home/wrolpi/.pgpass <<'EOF' -127.0.0.1:5432:gis:_renderd:wrolpi -127.0.0.1:5432:wrolpi:wrolpi:wrolpi -EOF -chown -R wrolpi:wrolpi /home/wrolpi /opt/wrolpi -chmod 0600 /home/wrolpi/.pgpass - -# Give WROLPi group a few privileged commands via sudo without password. -cat >/etc/sudoers.d/90-wrolpi <<'EOF' -%wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli,/usr/bin/cpufreq-set -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart renderd.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop renderd.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start renderd.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-kiwix.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-kiwix.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-kiwix.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-api.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-api.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-api.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-app.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-app.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-app.service -%wrolpi ALL= NOPASSWD:/opt/wrolpi/scripts/import_map.sh -%wrolpi ALL= NOPASSWD:/usr/sbin/shutdown -EOF -chmod 0440 /etc/sudoers.d/90-wrolpi -# Verify this new file is valid. -visudo -c -f /etc/sudoers.d/90-wrolpi - -# Create the media directory. This should be mounted by the maintainer. -[ -d /media/wrolpi ] || mkdir /media/wrolpi -chown wrolpi:wrolpi /media/wrolpi - -# Install the systemd services -cp /opt/wrolpi/etc/raspberrypios/wrolpi-api.service /etc/systemd/system/ -cp /opt/wrolpi/etc/raspberrypios/wrolpi-app.service /etc/systemd/system/ -cp /opt/wrolpi/etc/raspberrypios/wrolpi-kiwix.service /etc/systemd/system/ -cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/ -/usr/bin/systemctl daemon-reload -systemctl enable wrolpi-api.service -systemctl enable wrolpi-app.service -systemctl enable wrolpi-kiwix.service -# Stop the services so the user has to start them again. We don't want to run outdated services when updating. -systemctl stop wrolpi-api.service -systemctl stop wrolpi-app.service -systemctl stop wrolpi-kiwix.service - -# Configure Postgresql. Do this after the API is stopped. -sudo -u postgres psql -c '\l' | grep wrolpi || ( - sudo -u postgres createuser -s wrolpi && - sudo -u postgres psql -c "alter user postgres password 'wrolpi'" && - sudo -u postgres psql -c "alter user wrolpi password 'wrolpi'" && - sudo -u postgres createdb -E UTF8 -O wrolpi wrolpi -) -# wrolpi user is superuser so they can import maps. -sudo -u postgres psql -c "alter user wrolpi with superuser" -# Initialize/upgrade the WROLPi database. -(cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 /opt/wrolpi/main.py db upgrade) - -# Install map only if that script hasn't finished. -[ -f /var/www/html/leaflet.css ] || /opt/wrolpi/scripts/install_map_debian_11.sh - -chown -R wrolpi:wrolpi /opt/wrolpi - -set +x - -ip=$(hostname -I | cut -d' ' -f1) -if [[ $ip == *":"* ]]; then - # Don't suggest the ipv6 address. - ip=$(hostname -I | cut -d' ' -f2) -fi - -echo " - -WROLPi has successfully been installed! - -Mount your external hard drive to /media/wrolpi if you have one. Change the -file permissions if necessary: - # sudo chown -R wrolpi:wrolpi /media/wrolpi - -Start the WROLPi services using: - # sudo systemctl start wrolpi.target - -then navigate to: http://${ip} - -Or, join to the Wifi hotspot: -SSID: WROLPi -Password: wrolpi hotspot - -When on the hotspot, WROLPi is accessible at http://192.168.0.1 -" diff --git a/scripts/install_debian_12.sh b/scripts/install_debian_12.sh new file mode 100755 index 00000000..d72c5aef --- /dev/null +++ b/scripts/install_debian_12.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +echo "Debian 12 install start: $(date '+%Y-%m-%d %H:%M:%S')" + +set -x +set -e + +# Update if we haven't updated in the last day. +[ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mtime -1)" ] && apt update +# Install dependencies. +apt-get install -y < /opt/wrolpi/debian-live-config/config/package-lists/wrolpi.list.chroot + +# App dependencies were installed. +node -v +npm -v + +# Install serve, and archiving tools. +single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@1.0.33 readability-extractor@0.0.6 carto@1.2.0 + +# Build React app in background job. +cd /opt/wrolpi/app || exit 5 +npm install || npm install || npm install || npm install # try install multiple times :( + +# Install python requirements in background job. +python3 -m venv /opt/wrolpi/venv +/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt + +# Get map dependencies. +[ -d /opt/openstreetmap-carto ] && chown -R wrolpi:wrolpi /opt/openstreetmap-carto +git clone https://github.com/lrnselfreliance/openstreetmap-carto.git /opt/openstreetmap-carto || : +sudo -u _renderd /bin/bash -c '(cd /opt/openstreetmap-carto && git fetch && git checkout master && git reset --hard origin/master && git pull --ff)' + +# Repair will install configs and restart services. +/opt/wrolpi/repair.sh + +set +x + +ip=$(hostname -I | cut -d' ' -f1) +if [[ $ip == *":"* ]]; then + # Don't suggest the ipv6 address. + ip=$(hostname -I | cut -d' ' -f2) +fi + +echo " + +WROLPi has successfully been installed! + +Mount your external hard drive to /media/wrolpi if you have one. Change the +file permissions if necessary: + # sudo chown -R wrolpi:wrolpi /media/wrolpi + +Start the WROLPi services using: + # sudo systemctl start wrolpi.target + +then navigate to: http://${ip} + +Or, join to the Wifi hotspot: +SSID: WROLPi +Password: wrolpi hotspot + +When on the hotspot, WROLPi is accessible at http://192.168.0.1 +" diff --git a/scripts/install_map_debian_11.sh b/scripts/install_map_debian_11.sh deleted file mode 100755 index 0bd1ff54..00000000 --- a/scripts/install_map_debian_11.sh +++ /dev/null @@ -1,67 +0,0 @@ -#! /usr/bin/env bash -# This script installs OpenStreetMap tile server. -# This server uses Apache2, renderd, and mod_tile to render and display map tiles. - -set -x -set -e - -# Update if we haven't updated in the last day. -[ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mtime -1)" ] && apt update -# Install map dependencies. -apt install -y libapache2-mod-tile renderd tar unzip wget bzip2 apache2 lua5.1 mapnik-utils python3-mapnik gdal-bin \ - fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted ttf-unifont osmium-tool python3-psycopg2 python3-yaml - -/usr/bin/npm install -g carto@1.2.0 -carto -v - -# Create `gis` database for map. -sudo -u postgres psql -c '\l' | grep gis || ( - sudo -u postgres createuser _renderd || : - sudo -u postgres psql -c "alter user _renderd password 'wrolpi'" || : - sudo -u postgres createdb -E UTF8 -O _renderd gis - sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis" || : - sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore" || : - echo "Created gis database" -) -sudo -u postgres psql -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd' -sudo -u postgres psql -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd' -sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO _renderd" - -# Install Debian 11 compatible mod_tile. -git clone -b switch2osm https://github.com/lrnselfreliance/mod_tile.git /tmp/mod_tile || : -# Install init/systemd config to start map service on startup. -cp /opt/wrolpi/etc/raspberrypios/renderd /etc/init.d/renderd -sudo chmod +x /etc/init.d/renderd -cp /tmp/mod_tile/debian/renderd.service /lib/systemd/system/ -chmod +x /etc/init.d/renderd - -# Initialize gis database. -[ -d /opt/openstreetmap-carto ] && chown -R wrolpi:wrolpi /opt/openstreetmap-carto -git clone https://github.com/lrnselfreliance/openstreetmap-carto.git /opt/openstreetmap-carto || : -chown -R _renderd:_renderd /opt/openstreetmap-carto -sudo -u _renderd /bin/bash -c '(cd /opt/openstreetmap-carto && git fetch && git checkout master && git reset --hard origin/master && git pull --ff)' -cd /opt/openstreetmap-carto -if [[ ! -f /opt/openstreetmap-carto/mapnik.xml || ! -s /opt/openstreetmap-carto/mapnik.xml ]]; then - carto project.mml >/opt/openstreetmap-carto/mapnik.xml -fi -chown -R _renderd:_renderd /opt/openstreetmap-carto - -# Reset map. Force it. -/opt/wrolpi/scripts/reset_map.sh -f - -cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf -# Configure Apache2 to listen on 8084. -cp /opt/wrolpi/etc/raspberrypios/ports.conf /etc/apache2/ports.conf -cp /opt/wrolpi/etc/debian11/000-default.conf /etc/apache2/sites-available/000-default.conf -# Copy Leaflet files to Apache's directory so they can be used offline. -cp /opt/wrolpi/etc/raspberrypios/index.html \ - /opt/wrolpi/modules/map/leaflet.js \ - /opt/wrolpi/modules/map/leaflet.css /var/www/html/ -chmod 644 /var/www/html/* - -systemctl enable renderd -systemctl stop renderd -systemctl start renderd - -systemctl stop apache2 -systemctl start apache2 diff --git a/scripts/install_map_raspberrypios.sh b/scripts/install_map_raspberrypios.sh deleted file mode 100755 index ae429a76..00000000 --- a/scripts/install_map_raspberrypios.sh +++ /dev/null @@ -1,76 +0,0 @@ -#! /usr/bin/env bash -# This script installs OpenStreetMap tile server. -# This server uses Apache2, renderd, and mod_tile to render and display map tiles. - -set -x -set -e - -# Update if we haven't updated in the last day. -[ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mtime -1)" ] && apt update -# Install map dependencies. -apt install -y libapache2-mod-tile libboost-all-dev git tar unzip wget bzip2 build-essential autoconf libtool libxml2-dev libgeos-dev \ - libgeos++-dev libpq-dev libbz2-dev libproj-dev protobuf-c-compiler libfreetype6-dev libtiff5-dev \ - libicu-dev libgdal-dev libcairo2-dev libcairomm-1.0-dev libagg-dev liblua5.2-dev ttf-unifont lua5.1 liblua5.1-0-dev \ - postgis postgresql-13-postgis-3 postgresql-13-postgis-3-scripts osm2pgsql gdal-bin libmapnik-dev mapnik-utils \ - python3-mapnik apache2 apache2-dev libcurl4-gnutls-dev libiniparser-dev libmemcached-dev librados-dev \ - fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted postgresql-contrib osmium-tool python3-psycopg2 python3-yaml - -/usr/bin/npm install -g carto@1.2.0 -carto -v - -# Create `gis` database for map. -sudo -u postgres psql -c '\l' | grep gis || ( - sudo -u postgres createuser _renderd || : - sudo -u postgres psql -c "alter user _renderd password 'wrolpi'" || : - sudo -u postgres createdb -E UTF8 -O _renderd gis - sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis" || : - sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore" || : - echo "Created gis database" -) -sudo -u postgres psql -d gis -c 'ALTER TABLE geography_columns OWNER TO _renderd' -sudo -u postgres psql -d gis -c 'ALTER TABLE geometry_columns OWNER TO _renderd' -sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO _renderd" - -# Install Raspberry Pi OS compatible mod_tile. -git clone -b switch2osm https://github.com/lrnselfreliance/mod_tile.git /tmp/mod_tile || : -# Install init/systemd config to start map service on startup. -cp /opt/wrolpi/etc/raspberrypios/renderd /etc/init.d/renderd -sudo chmod +x /etc/init.d/renderd -cp /tmp/mod_tile/debian/renderd.service /lib/systemd/system/ -chmod +x /etc/init.d/renderd - -# Initialize gis database. -[ -d /opt/openstreetmap-carto ] && chown -R wrolpi:wrolpi /opt/openstreetmap-carto -git clone https://github.com/lrnselfreliance/openstreetmap-carto.git /opt/openstreetmap-carto || : -chown -R _renderd:_renderd /opt/openstreetmap-carto -sudo -u _renderd /bin/bash -c '(cd /opt/openstreetmap-carto && git fetch && git checkout master && git reset --hard origin/master && git pull --ff)' -cd /opt/openstreetmap-carto -if [[ ! -f /opt/openstreetmap-carto/mapnik.xml || ! -s /opt/openstreetmap-carto/mapnik.xml ]]; then - carto project.mml >/opt/openstreetmap-carto/mapnik.xml -fi -chown -R _renderd:_renderd /opt/openstreetmap-carto - -# Reset map. Force it. -/opt/wrolpi/scripts/reset_map.sh -f - -cp /opt/wrolpi/etc/raspberrypios/renderd.conf /etc/renderd.conf -# Enable mod_tile. -cp /opt/wrolpi/etc/raspberrypios/mod_tile.conf /etc/apache2/conf-available/mod_tile.conf - -/usr/sbin/a2enconf mod_tile - -# Configure Apache2 to listen on 8084. -cp /opt/wrolpi/etc/raspberrypios/ports.conf /etc/apache2/ports.conf -cp /opt/wrolpi/etc/raspberrypios/000-default.conf /etc/apache2/sites-available/000-default.conf -# Copy Leaflet files to Apache's directory so they can be used offline. -cp /opt/wrolpi/etc/raspberrypios/index.html \ - /opt/wrolpi/modules/map/leaflet.js \ - /opt/wrolpi/modules/map/leaflet.css /var/www/html/ -chmod 644 /var/www/html/* - -systemctl enable renderd -systemctl stop renderd -systemctl start renderd - -systemctl stop apache2 -systemctl start apache2 diff --git a/scripts/install_raspberrypios.sh b/scripts/install_raspberrypios.sh index f180de31..54f863a0 100755 --- a/scripts/install_raspberrypios.sh +++ b/scripts/install_raspberrypios.sh @@ -7,138 +7,30 @@ set -e # Update if we haven't updated in the last day. [ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mtime -1)" ] && apt update # Install dependencies. -apt-get install -y \ - apt-transport-https \ - ca-certificates \ - calibre \ - chromium \ - chromium-driver \ - cpufrequtils \ - curl \ - ffmpeg \ - gcc \ - gnupg-agent \ - hostapd \ - htop \ - iotop \ - kiwix \ - kiwix-tools \ - libpq-dev \ - netcat \ - network-manager \ - nginx-doc \ - nginx-full \ - postgresql-13 \ - python3-dev \ - python3-doc \ - python3-full \ - python3-pip \ - python3-venv \ - software-properties-common \ - tmux \ - vim \ - zim-tools +apt-get install -y < /opt/wrolpi/pi-gen/stage2/04-wrolpi/01-packages -# Install npm. -npm --version || curl -fsSL https://deb.nodesource.com/setup_18.x | bash - -apt-get install -y nodejs +# App dependencies were installed. node -v npm -v # Install serve, and archiving tools. -sudo npm i -g serve@12.0.1 single-file-cli@1.0.33 readability-extractor@0.0.6 +single-file --version || sudo npm i -g serve@12.0.1 single-file-cli@1.0.33 readability-extractor@0.0.6 carto@1.2.0 # Build React app in background job. cd /opt/wrolpi/app || exit 5 npm install || npm install || npm install || npm install # try install multiple times :( -npm run build & # Install python requirements in background job. python3 -m venv /opt/wrolpi/venv -/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt & +/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt -wait $(jobs -p) +# Get map dependencies. +[ -d /opt/openstreetmap-carto ] && chown -R wrolpi:wrolpi /opt/openstreetmap-carto +git clone https://github.com/lrnselfreliance/openstreetmap-carto.git /opt/openstreetmap-carto || : +sudo -u _renderd /bin/bash -c '(cd /opt/openstreetmap-carto && git fetch && git checkout master && git reset --hard origin/master && git pull --ff)' -# Install the WROLPi nginx config over the default nginx config. -cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf -cp /opt/wrolpi/50x.html /var/www/50x.html -/usr/sbin/nginx -s reload - -# Create the WROLPi user -grep wrolpi: /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)" -cat >/home/wrolpi/.pgpass <<'EOF' -127.0.0.1:5432:gis:_renderd:wrolpi -127.0.0.1:5432:wrolpi:wrolpi:wrolpi -EOF -chown -R wrolpi:wrolpi /home/wrolpi /opt/wrolpi -chmod 0600 /home/wrolpi/.pgpass - -# Allow pi user to access database. -[ -d /home/pi ] && ( - cat >/home/pi/.pgpass <<'EOF' -127.0.0.1:5432:gis:_renderd:wrolpi -127.0.0.1:5432:wrolpi:wrolpi:wrolpi -EOF - chown -R pi:pi /home/pi - chmod 0600 /home/pi/.pgpass -) - -# Give WROLPi group a few privileged commands via sudo without password. -cat >/etc/sudoers.d/90-wrolpi <<'EOF' -%wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli,/usr/bin/cpufreq-set -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart renderd.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop renderd.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start renderd.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-kiwix.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-kiwix.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-kiwix.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-api.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-api.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-api.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl restart wrolpi-app.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl stop wrolpi-app.service -%wrolpi ALL= NOPASSWD:/usr/bin/systemctl start wrolpi-app.service -%wrolpi ALL= NOPASSWD:/opt/wrolpi/scripts/import_map.sh -%wrolpi ALL= NOPASSWD:/usr/sbin/shutdown -EOF -chmod 0440 /etc/sudoers.d/90-wrolpi -# Verify this new file is valid. -visudo -c -f /etc/sudoers.d/90-wrolpi - -# Create the media directory. This should be mounted by the maintainer. -[ -d /media/wrolpi ] || mkdir /media/wrolpi -chown wrolpi:wrolpi /media/wrolpi - -# Install the systemd services -cp /opt/wrolpi/etc/raspberrypios/wrolpi-api.service /etc/systemd/system/ -cp /opt/wrolpi/etc/raspberrypios/wrolpi-app.service /etc/systemd/system/ -cp /opt/wrolpi/etc/raspberrypios/wrolpi-kiwix.service /etc/systemd/system/ -cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/ -/usr/bin/systemctl daemon-reload -systemctl enable wrolpi-api.service -systemctl enable wrolpi-app.service -systemctl enable wrolpi-kiwix.service -# Stop the services so the user has to start them again. We don't want to run outdated services when updating. -systemctl stop wrolpi-api.service -systemctl stop wrolpi-app.service -systemctl stop wrolpi-kiwix.service - -# Configure Postgresql. Do this after the API is stopped. -sudo -u postgres psql -c '\l' | grep wrolpi || ( - sudo -u postgres createuser -s wrolpi && - sudo -u postgres psql -c "alter user postgres password 'wrolpi'" && - sudo -u postgres psql -c "alter user wrolpi password 'wrolpi'" && - sudo -u postgres createdb -E UTF8 -O wrolpi wrolpi -) -# wrolpi user is superuser so they can import maps. -sudo -u postgres psql -c "alter user wrolpi with superuser" -# Initialize/upgrade the WROLPi database. -(cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 /opt/wrolpi/main.py db upgrade) - -# Install map only if that script hasn't finished. -[ -f /var/www/html/leaflet.css ] || /opt/wrolpi/scripts/install_map_raspberrypios.sh - -chown -R wrolpi:wrolpi /opt/wrolpi +# Repair will install configs and restart services. +/opt/wrolpi/repair.sh set +x diff --git a/scripts/reset_api_db.sh b/scripts/reset_api_db.sh new file mode 100755 index 00000000..da6646f6 --- /dev/null +++ b/scripts/reset_api_db.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +source /opt/wrolpi/wrolpi/scripts/lib.sh + +# Re-execute this script if it wasn't called with sudo. +if [ $EUID != 0 ]; then + sudo "$0" "$@" + exit $? +fi + +if psql -l 2>/dev/null | grep wrolpi >/dev/null; then + yes_or_no "Are you sure you want to reset the API database? All data will be lost." || exit 0 +fi + +set -x + +systemctl stop wrolpi-api + +# Delete the WROLPi API DB, if it exists. +sudo -u postgres dropdb wrolpi +sudo -u postgres dropuser wrolpi + +/bin/bash /opt/wrolpi/scripts/initialize_api_db.sh diff --git a/scripts/reset_map.sh b/scripts/reset_map.sh index cdfb4bb3..6cb2bbc2 100755 --- a/scripts/reset_map.sh +++ b/scripts/reset_map.sh @@ -1,59 +1,15 @@ #!/usr/bin/env bash -# Use 1/4 of the RAM to import. 1/2 causes crashes on RPi. -RAM_KB=$(grep MemTotal /proc/meminfo | awk '{print $2}') -MAX_CACHE=$((RAM_KB / 1024 / 4)) +source /opt/wrolpi/wrolpi/scripts/lib.sh -function yes_or_no { - while true; do - read -p "$* [y/n]:" yn - case $yn in - [Yy]*) return 0 ;; - [Nn]*) return 1 ;; - esac - done -} - -Help() { - # Display Help - echo "Reset the Map cache/database" - echo - echo "Syntax: reset_map.sh [-h] [-f]" - echo "options:" - echo "h Print this help." - echo "f Reset without prompt." - echo -} - -FORCE=false -while getopts ":hf" option; do - case $option in - h) # display Help - Help - exit - ;; - f) - FORCE=true - ;; - *) # invalid argument(s) - echo "Error: Invalid option" - exit 1 - ;; - esac -done - -if [ "${FORCE}" == false ]; then - yes_or_no "Are you sure you want to delete the map database and cache? This can't be undone!" || exit 1 +# Re-execute this script if it wasn't called with sudo. +if [ $EUID != 0 ]; then + sudo "$0" "$@" + exit $? fi -if [[ ! -f /usr/lib/apache2/modules/mod_tile.so || - ! -f /opt/openstreetmap-carto/mapnik.xml || - ! -s /opt/openstreetmap-carto/mapnik.xml || - ! -f /opt/openstreetmap-carto/scripts/get-external-data.py || - ! -f /opt/openstreetmap-carto/openstreetmap-carto.style || - ! -f /opt/openstreetmap-carto/indexes.sql ]]; then - echo "The map has not been installed. Run install.sh" - exit 2 +if psql -l 2>/dev/null | grep gis >/dev/null; then + yes_or_no "Are you sure you want to delete the map database and cache? This can't be undone!" || exit 0 fi set -e @@ -61,27 +17,14 @@ set -x systemctl stop renderd -if [ -d /var/lib/mod_tile ]; then - rm -rf /var/lib/mod_tile/* -else - mkdir /var/lib/mod_tile -fi -chown -R _renderd:_renderd /var/lib/mod_tile - sudo -u postgres dropdb gis || : - -sudo -u postgres createdb -E UTF8 -O _renderd gis -sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis;" || : -sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore;" || : -sudo -u postgres psql -d gis -c "ALTER TABLE geometry_columns OWNER TO wrolpi;" -sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO wrolpi;" +sudo -u postgres dropuser _renderd || : # Reset "imported" status of any map files. sudo -u postgres psql -d wrolpi -c "UPDATE map_file SET imported=false" -bash /opt/wrolpi/scripts/initialize_map.sh +yes | /opt/wrolpi/scripts/initialize_map_db.sh -systemctl enable renderd -systemctl start renderd +set +x echo "Map has been reset." diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index 30696f58..9aeba416 100755 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -1,15 +1,17 @@ #! /usr/bin/env bash +# Installs any new dependencies of the App and API. Uses internet. -# Update systemd files. -cp /opt/wrolpi/etc/raspberrypios/wrolpi-api.service /etc/systemd/system/ -/usr/bin/systemctl daemon-reload +set -e +set -x -# Upgrade the API and App. -/bin/bash /opt/wrolpi/scripts/build_api_and_app.sh +# Install any App dependencies. +cd /opt/wrolpi/app || exit 1 +npm install || npm install || npm install || npm install # try install multiple times :( -# Update nginx configs -cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf -cp /opt/wrolpi/50x.html /var/www/50x.html -/usr/sbin/nginx -s reload +# Install any new Python requirements. +/opt/wrolpi/venv/bin/pip3 install -r /opt/wrolpi/requirements.txt +# Upgrade the WROLPi database. +(cd /opt/wrolpi && /opt/wrolpi/venv/bin/python3 /opt/wrolpi/main.py db upgrade) -chown -R wrolpi:wrolpi /opt/wrolpi +# Install any configs, restart services. +/opt/wrolpi/repair.sh diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 98c905ec..cc4b3a11 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -172,6 +172,12 @@ class Download(ModelHelper, Base): # noqa return list(filterfalse(excluded, urls)) return urls + def add_to_skip_list(self): + if self.manager: + self.manager.add_to_skip_list(self.url) + else: + raise RuntimeError(f'Cannot add {self} to skip list because I do not have a manager.') + class Downloader: name: str = None @@ -1059,6 +1065,12 @@ class DownloadManager: session.execute('DELETE FROM download WHERE id = ANY(:ids)', {'ids': ids}) session.commit() + @staticmethod + def is_skipped(*urls: str) -> bool: + if skip_list := get_download_manager_config().skip_urls: + return all(i in skip_list for i in urls) + return False + @staticmethod def add_to_skip_list(*urls: str): get_download_manager_config().skip_urls = list(set(get_download_manager_config().skip_urls) | set(urls)) diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 7ea9d4e8..61394f5a 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -213,13 +213,12 @@ async def test__upsert_files(test_session, make_files_structure, test_directory, }) video_file = video_file.rename(test_directory / 'video.mp4') srt_file3 = srt_file3.rename(test_directory / 'video.en.srt') - foo_mtime = from_timestamp(srt_file3.stat().st_mtime) # All files are found because they are in this refresh request, or in the `dir1` directory. idempotency = now() lib._upsert_files([video_file, srt_file3, bar, baz], idempotency) assert_file_groups([ - {'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': False, + {'primary_path': video_file, 'idempotency': idempotency, 'indexed': False, 'files': [ {'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'}, {'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'}, @@ -239,14 +238,14 @@ async def test__upsert_files(test_session, make_files_structure, test_directory, test_session.query(FileGroup).filter_by(primary_path=video_file).one().indexed = True test_session.commit() assert_file_groups( - [{'primary_path': str(video_file), 'idempotency': idempotency, 'modification_datetime': foo_mtime, + [{'primary_path': str(video_file), 'idempotency': idempotency, 'indexed': True}], assert_count=False) # Only modified files need to be re-indexed. lib._upsert_files([video_file, srt_file3, bar, baz], idempotency) assert_file_groups([ - {'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': True, + {'primary_path': video_file, 'idempotency': idempotency, 'indexed': True, 'files': [ {'path': srt_file3, 'size': 951, 'suffix': '.en.srt', 'mimetype': 'text/srt'}, {'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'}, @@ -266,7 +265,7 @@ async def test__upsert_files(test_session, make_files_structure, test_directory, assert len(video_file_group.files) == 1, 'SRT file was not removed from files' assert_file_groups([ # Video is no longer indexed because SRT was removed. - {'primary_path': video_file, 'idempotency': idempotency, 'modification_datetime': foo_mtime, 'indexed': False, + {'primary_path': video_file, 'idempotency': idempotency, 'indexed': False, 'files': [{'path': video_file, 'size': 1056318, 'suffix': '.mp4', 'mimetype': 'video/mp4'}]}, {'primary_path': bar, 'idempotency': idempotency, 'indexed': False, 'files': [{'path': bar, 'size': 0, 'suffix': '.txt', 'mimetype': 'inode/x-empty'}], diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 6f6de3a0..7018b70b 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -228,8 +228,6 @@ def valid_regex(_: Request, body: schema.RegexRequest): @validate(schema.DownloadRequest) @wrol_mode_check async def post_download(_: Request, body: schema.DownloadRequest): - # URLs are provided in a textarea, lets split all lines. - urls = [i.strip() for i in str(body.urls).strip().splitlines()] downloader = download_manager.get_downloader_by_name(body.downloader) if not downloader: raise InvalidDownload(f'Cannot find downloader with name {body.downloader}') @@ -248,11 +246,11 @@ async def post_download(_: Request, body: schema.DownloadRequest): depth=body.depth) if body.frequency: - download_manager.recurring_download(urls[0], body.frequency, downloader_name=body.downloader, + download_manager.recurring_download(body.urls[0], body.frequency, downloader_name=body.downloader, sub_downloader_name=body.sub_downloader, reset_attempts=True, settings=settings) else: - download_manager.create_downloads(urls, downloader_name=body.downloader, reset_attempts=True, + download_manager.create_downloads(body.urls, downloader_name=body.downloader, reset_attempts=True, sub_downloader_name=body.sub_downloader, settings=settings) if download_manager.disabled.is_set() or download_manager.stopped.is_set(): # Downloads are disabled, warn the user. diff --git a/wrolpi/schema.py b/wrolpi/schema.py index c68f5ca5..35a7bc0d 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -1,6 +1,8 @@ from dataclasses import dataclass, field from typing import Optional, List +from wrolpi.errors import InvalidDownload + @dataclass class SettingsObject: @@ -50,7 +52,7 @@ class EchoResponse: @dataclass class DownloadRequest: - urls: str + urls: List[str] downloader: str frequency: Optional[int] = None sub_downloader: Optional[str] = None @@ -61,6 +63,13 @@ class DownloadRequest: depth: Optional[int] = None max_pages: Optional[int] = None + def __post_init__(self): + urls = [j for i in self.urls if (j := i.strip())] + if not urls: + raise InvalidDownload(f'urls cannot be empty') + # Get unique URLs, preserve order. + self.urls = list(dict.fromkeys(urls)) + @dataclass class JSONErrorResponse: diff --git a/wrolpi/scripts/lib.sh b/wrolpi/scripts/lib.sh new file mode 100755 index 00000000..4630bd7b --- /dev/null +++ b/wrolpi/scripts/lib.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Shared bash scripting functionality. This is not meant to be run by itself. + +function yes_or_no { + while true; do + read -p "$* [y/n]:" yn + case $yn in + [Yy]*) return 0 ;; + [Nn]*) return 1 ;; + esac + done +} + +# The full path to the WROLPi project directory (typically "/opt/wrolpi") +PROJECT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +PROJECT_DIR=$(dirname "${PROJECT_DIR}") +PROJECT_DIR=$(dirname "${PROJECT_DIR}") + +rpi=false +debian12=false +if (grep 'Raspberry Pi' /proc/cpuinfo >/dev/null); then + rpi=true +fi +if (grep 'PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"' /etc/os-release >/dev/null); then + debian12=true +fi diff --git a/wrolpi/status.py b/wrolpi/status.py index 97b09b58..cd2bd13d 100755 --- a/wrolpi/status.py +++ b/wrolpi/status.py @@ -143,14 +143,15 @@ async def get_cpu_info() -> CPUInfo: stderr=subprocess.PIPE, ) stdout, stderr = await proc.communicate() - if proc.returncode != 0: + match = TOP_REGEX.search(stdout.decode()) + if proc.returncode != 0 or not match: logger.warning(f'Unable to get CPU top with exit {proc.returncode}') percent = None else: - percent = int(Decimal(TOP_REGEX.search(stdout.decode()).groups()[0])) + percent = int(Decimal(match.groups()[0])) if not TEMPERATURE_PATH.is_file(): - logger.warning(f'CPU temperature file does not exist!') + warn_once(f'CPU temperature file does not exist!') return CPUInfo(percent=percent) try: diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index c896f7a2..7fc38b59 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -321,7 +321,7 @@ async def test_process_runner_timeout(test_directory): async def test_crud_download(test_async_client, test_session, test_download_manager, test_downloader): """Test the ways that Downloads can be created.""" body = dict( - urls='https://example.com', + urls=['https://example.com', ], downloader=test_downloader.name, frequency=DownloadFrequency.weekly, excluded_urls='example.org,something', diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index a8470952..d9fab2cb 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -366,7 +366,7 @@ async def test_get_status(test_async_client, test_session): def test_post_download(test_session, test_client, test_download_manager_config): - """Test creating once-downlaods and recurring downloads.""" + """Test creating once-downloads and recurring downloads.""" async def queue_downloads(*a, **kw): pass @@ -374,7 +374,7 @@ def test_post_download(test_session, test_client, test_download_manager_config): with mock.patch('wrolpi.downloader.DownloadManager.queue_downloads', queue_downloads): # Create a single recurring Download. content = dict( - urls='https://example.com/1\n', + urls=['https://example.com/1', ], frequency=1_000, downloader='archive', ) @@ -385,7 +385,7 @@ def test_post_download(test_session, test_client, test_download_manager_config): # Create to once-downloads. content = dict( - urls='https://example.com/2\nhttps://example.com/3\n', + urls=['https://example.com/2', 'https://example.com/3'], downloader='archive', ) request, response = test_client.post('/api/download', content=json.dumps(content)) @@ -403,6 +403,19 @@ def test_get_downloaders(test_client): 'No downloaders returned' +@pytest.mark.asyncio +async def test_empty_post_download(test_async_client): + """Cannot have empty download URLs""" + content = dict( + urls=[''], # Cannot have empty URLs + frequency=1_000, + downloader='archive', + ) + request, response = await test_async_client.post('/api/download', content=json.dumps(content)) + assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.json['code'] == 'INVALID_DOWNLOAD' + + @pytest.mark.asyncio async def test_restart_download(test_session, test_async_client, test_download_manager, test_downloader): """A Download can be restarted.""" diff --git a/wrolpi/version.txt b/wrolpi/version.txt index db044fb1..fc7d045f 100644 --- a/wrolpi/version.txt +++ b/wrolpi/version.txt @@ -1 +1 @@ -0.11.4-beta \ No newline at end of file +0.12.0-beta \ No newline at end of file