From 849a3fd43f644e07199d5eca232d556ddb7b6faf Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 16 Mar 2022 18:09:02 -0600 Subject: [PATCH] Creating debian install scripts. Modifying Ubuntu 20.04 install script to work with Debian 11. Should not need to compile mod_tile. Uncommenting branch checkout. Not removing nodejs. Not installing nodejs. Installing nodejs instead of npm?? Using debian 11 npm sources. Installing archive tools using git+https Updating carto location. Using global renderd.conf in apache config. Using -I to get debian ip. Updating more file locations for debian 11. Checking alternative locations for single-file and readability-extractor --- install.sh | 24 ++- modules/archive/lib.py | 4 + scripts/install_debian_11.sh | 103 ++++++++++ scripts/install_map_debian_11.sh | 187 ++++++++++++++++++ ...all_map.sh => install_map_ubuntu_20.04.sh} | 0 ....04_install.sh => install_ubuntu_20.04.sh} | 5 +- 6 files changed, 315 insertions(+), 8 deletions(-) create mode 100755 scripts/install_debian_11.sh create mode 100755 scripts/install_map_debian_11.sh rename scripts/{ubuntu_20.04_install_map.sh => install_map_ubuntu_20.04.sh} (100%) rename scripts/{ubuntu_20.04_install.sh => install_ubuntu_20.04.sh} (98%) diff --git a/install.sh b/install.sh index 082cbc8d..b548c933 100755 --- a/install.sh +++ b/install.sh @@ -30,8 +30,17 @@ while getopts ":hb:" option; do esac done -if (! grep 'Raspberry Pi' /proc/cpuinfo >/dev/null); then - echo 'This script can only be run on a Raspberry Pi!' && exit 2 +rpi=false +debian11=false +if (grep 'Raspberry Pi' /proc/cpuinfo >/dev/null); then + rpi=true +fi +if (grep 'PRETTY_NAME="Debian GNU/Linux 11 (bullseye)"' /etc/os-release >/dev/null); then + debian11=true +fi + +if [[ ${rpi} == false && ${debian11} == false ]]; then + echo 'This script can only be run on a Raspberry Pi or Debian 11!' && exit 2 fi set -x @@ -46,10 +55,15 @@ git --version git clone https://github.com/lrnselfreliance/wrolpi.git /opt/wrolpi || : (cd /opt/wrolpi && git fetch && git checkout "${BRANCH}" && git reset --hard origin/"${BRANCH}") || exit 4 -/opt/wrolpi/scripts/ubuntu_20.04_install.sh 2>&1 | tee /opt/wrolpi/install.log -install_code=$? +if [ ${rpi} == true ]; then + /opt/wrolpi/scripts/install_ubuntu_20.04.sh 2>&1 | tee /opt/wrolpi/install.log + install_code=${PIPESTATUS[0]} +elif [ ${debian11} == true ]; then + /opt/wrolpi/scripts/install_debian_11.sh 2>&1 | tee /opt/wrolpi/install.log + install_code=${PIPESTATUS[0]} +fi set +x echo "Install end $(date '+%Y-%m-%d %H:%M:%S')" >>/opt/wrolpi/install.log -exit ${install_code} +exit "${install_code}" diff --git a/modules/archive/lib.py b/modules/archive/lib.py index b5bc2cea..693bbe0b 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -152,6 +152,8 @@ def request_archive(url: str): def local_singlefile(url: str): """Run the single-file executable to create an HTML file archive.""" single_file_path = pathlib.Path('/usr/bin/single-file') + if not single_file_path.is_file(): + single_file_path = pathlib.Path('/usr/local/bin/single-file') if not single_file_path.is_file(): raise FileNotFoundError(f'single-file not found at {single_file_path}') @@ -190,6 +192,8 @@ def local_extract_readability(path: str, url: str) -> dict: """Extract the readability from an HTML file, typically from single-file.""" logger.info(f'readability for {url}') readability_path = pathlib.Path('/usr/bin/readability-extractor') + if not readability_path.is_file(): + readability_path = pathlib.Path('/usr/local/bin/readability-extractor') if not readability_path.is_file(): raise FileNotFoundError(f'Readability extractor not found at {readability_path}') diff --git a/scripts/install_debian_11.sh b/scripts/install_debian_11.sh new file mode 100755 index 00000000..13b79d84 --- /dev/null +++ b/scripts/install_debian_11.sh @@ -0,0 +1,103 @@ +#!/bin/bash +echo "Debian 11 install start: $(date '+%Y-%m-%d %H:%M:%S')" + +set -x +set -e + +[ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mtime -1)" ] && apt update +# Install dependencies +apt install -y apt-transport-https ca-certificates curl gnupg-agent gcc libpq-dev software-properties-common \ + postgresql-13 nginx-full nginx-doc python3-full python3-dev python3-doc python3-venv htop vim ffmpeg hostapd \ + texlive-latex-base texlive-latex-extra chromium chromium-driver cpufrequtils network-manager npm + +# Install Archiving tools. +npm install -g 'git+https://github.com/gildas-lormeau/SingleFile' +npm install -g 'git+https://github.com/pirate/readability-extractor' + +# Build React app +[[ ! -f /usr/local/bin/serve || ! -f /usr/bin/serve ]] && npm -g install serve +cd /opt/wrolpi/app || exit 5 +npm install || npm install || npm install || npm install # try install multiple times +npm run build + +# Setup the virtual environment that main.py expects +pip3 --version || ( + # If no pip, install pip + curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py && + python3 /tmp/get-pip.py +) + +# Install python requirements files +pip3 install -r /opt/wrolpi/requirements.txt + +# Install the WROLPi nginx config over the default nginx config. +cp /opt/wrolpi/nginx.conf /etc/nginx/nginx.conf +cp /opt/wrolpi/50x.html /var/www/50x.html +/usr/sbin/nginx -s reload + +# Create the WROLPi user +grep wrolpi /etc/passwd || useradd -md /home/wrolpi wrolpi -s "$(command -v bash)" +chown -R wrolpi:wrolpi /opt/wrolpi + +# Give WROLPi group a few privileged commands via sudo without password. +cat >/etc/sudoers.d/90-wrolpi <<'EOF' +%wrolpi ALL=(ALL) NOPASSWD:/usr/bin/nmcli,/usr/bin/cpufreq-set +EOF +chmod 660 /etc/sudoers.d/90-wrolpi +# Verify this new file is valid. +visudo -c -f /etc/sudoers.d/90-wrolpi + +# Create the media directory. This should be mounted by the maintainer. +[ -d /media/wrolpi ] || mkdir /media/wrolpi +chown wrolpi:wrolpi /media/wrolpi + +# Install the systemd services +cp /opt/wrolpi/systemd/wrolpi-api.service /etc/systemd/system/ +cp /opt/wrolpi/systemd/wrolpi-app.service /etc/systemd/system/ +sed -i -e 's/^ExecStart=.*/ExecStart=\/usr\/local\/bin\/serve/' /etc/systemd/system/wrolpi-app.service +cp /opt/wrolpi/systemd/wrolpi.target /etc/systemd/system/ +/usr/bin/systemctl daemon-reload +systemctl enable wrolpi-api.service +systemctl enable wrolpi-app.service +# Stop the services so the user has to start them again. We don't want to run outdated services when updating. +systemctl stop wrolpi-api.service +systemctl stop wrolpi-app.service + +# Configure Postgresql. Do this after the API is stopped. +sudo -u postgres psql -c '\l' | grep wrolpi || ( + sudo -u postgres createuser wrolpi && + sudo -u postgres psql -c "alter user postgres password 'wrolpi'" && + sudo -u postgres psql -c "alter user wrolpi password 'wrolpi'" && + sudo -u postgres createdb -E UTF8 -O wrolpi wrolpi +) +# Initialize/upgrade the WROLPi database. +(cd /opt/wrolpi && /usr/bin/python3 /opt/wrolpi/main.py db upgrade) + +# Run the map installation script. +/opt/wrolpi/scripts/install_map_debian_11.sh + +set +x + +ip=$(hostname -I | cut -d' ' -f1) +if [[ $ip == *":"* ]]; then + # Don't suggest the ipv6 address. + ip=$(hostname -I | cut -d' ' -f2) +fi + +echo " + +WROLPi has successfully been installed! + +Mount your external hard drive to /media/wrolpi if you have one. + +Start the WROLPi services using: + # sudo systemctl start wrolpi.target + +then navigate to: http://${ip} + +Or, join to the Wifi hotspot: +SSID: WROLPi +Password: wrolpi hotspot + +When on the hotspot, WROLPi is accessible at http://192.168.0.1 +" diff --git a/scripts/install_map_debian_11.sh b/scripts/install_map_debian_11.sh new file mode 100755 index 00000000..d2f1e14a --- /dev/null +++ b/scripts/install_map_debian_11.sh @@ -0,0 +1,187 @@ +#! /usr/bin/bash +# This script installs OpenStreetMap tile server. +# This server uses Apache2, renderd and mod_tile to render and display map tiles. + +set -x +set -e + +# Update if we haven't updated in the last day. +[ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mtime -1)" ] && apt update +# Install map dependencies. +apt install -y libapache2-mod-tile renderd tar unzip wget bzip2 apache2 lua5.1 mapnik-utils python3-mapnik gdal-bin \ + fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted ttf-unifont + +/usr/bin/npm install -g carto +/usr/local/bin/carto -v + +# Create `gis` database for map. +sudo -u postgres psql -c '\l' | grep gis || sudo -u postgres createdb -E UTF8 -O wrolpi gis +sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis;" || : +sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore;" || : +sudo -u postgres psql -d gis -c "ALTER TABLE geometry_columns OWNER TO wrolpi;" +sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO wrolpi;" + +# Install Debian 11 compatible mod_tile. +git clone -b switch2osm https://github.com/lrnselfreliance/mod_tile.git /tmp/mod_tile || : +# Install init/systemd config to start map service on startup. +cp /tmp/mod_tile/debian/renderd.init /etc/init.d/renderd +sudo chmod +x /etc/init.d/renderd +sed -i -e 's/^RUNASUSER.*/RUNASUSER=wrolpi/' /etc/init.d/renderd +sed -i -e 's/^DAEMON=.*/DAEMON=\/usr\/bin\/$NAME/' /etc/init.d/renderd +sed -i -e 's/^DAEMON_ARGS=.*/DAEMON_ARGS="-c \/etc\/renderd.conf"/' /etc/init.d/renderd +cp /tmp/mod_tile/debian/renderd.service /lib/systemd/system/ +chmod +x /etc/init.d/renderd +[ ! -d /var/lib/mod_tile ] && mkdir /var/lib/mod_tile +chown wrolpi /var/lib/mod_tile + +# Initialize gis database. +git clone https://github.com/lrnselfreliance/openstreetmap-carto.git /opt/openstreetmap-carto || : +chown -R wrolpi:wrolpi /opt/openstreetmap-carto +cd /opt/openstreetmap-carto +if [[ ! -f /opt/openstreetmap-carto/mapnik.xml || ! -s /opt/openstreetmap-carto/mapnik.xml ]]; then + /usr/local/bin/carto project.mml >/opt/openstreetmap-carto/mapnik.xml +fi +# Initialize indexes and global polygons. +sudo -u wrolpi /opt/openstreetmap-carto/scripts/get-external-data.py -d gis -U wrolpi +# Use D.C. to initialized DB because it is so small. +wget --continue https://download.geofabrik.de/north-america/us/district-of-columbia-latest.osm.pbf \ + -O /tmp/district-of-columbia-latest.osm.pbf +# Run import in "create" mode so basic tables will be created. +sudo -u wrolpi osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script \ + /opt/openstreetmap-carto/openstreetmap-carto.lua -C 2000 --number-processes 4 \ + -S /opt/openstreetmap-carto/openstreetmap-carto.style /tmp/district-of-columbia-latest.osm.pbf +sudo -u wrolpi psql -d gis -f /opt/openstreetmap-carto/indexes.sql + +cat >/etc/renderd.conf <<'EOF' +[renderd] +stats_file=/run/renderd/renderd.stats +socketname=/run/renderd/renderd.sock +num_threads=4 +tile_dir=/var/cache/renderd/tiles + +[mapnik] +plugins_dir=/usr/lib/mapnik/3.1/input +font_dir=/usr/share/fonts/truetype +font_dir_recurse=true + +[ajt] +URI=/hot/ +TILEDIR=/var/lib/mod_tile +XML=/opt/openstreetmap-carto/mapnik.xml +HOST=localhost +TILESIZE=256 +MAXZOOM=20 + +EOF + +# Configure Apache2 to listen on 8084. +cat >/etc/apache2/ports.conf <<'EOF' +# If you just change the port or add more ports here, you will likely also +# have to change the VirtualHost statement in +# /etc/apache2/sites-enabled/000-default.conf + +Listen 8084 + + + Listen 443 + + + + Listen 443 + + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet +EOF + +cat >/etc/apache2/sites-available/000-default.conf <<'EOF' + + # The ServerName directive sets the request scheme, hostname and port that + # the server uses to identify itself. This is used when creating + # redirection URLs. In the context of virtual hosts, the ServerName + # specifies what hostname must appear in the request's Host: header to + # match this virtual host. For the default virtual host (this file) this + # value is not decisive as it is used as a last resort host regardless. + # However, you must set it for any further virtual host explicitly. + #ServerName www.example.com + + ServerAdmin webmaster@localhost + + LoadTileConfigFile /etc/renderd.conf + ModTileRenderdSocketName /var/run/renderd/renderd.sock + # Timeout before giving up for a tile to be rendered + ModTileRequestTimeout 0 + # Timeout before giving up for a tile to be rendered that is otherwise missing + ModTileMissingRequestTimeout 30 + + DocumentRoot /var/www/html + + # Available loglevels: trace8, ..., trace1, debug, info, notice, warn, + # error, crit, alert, emerg. + # It is also possible to configure the loglevel for particular + # modules, e.g. + #LogLevel info ssl:warn + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + # For most configuration files from conf-available/, which are + # enabled or disabled at a global level, it is possible to + # include a line for only one particular virtual host. For example the + # following line enables the CGI configuration for this host only + # after it has been globally disabled with "a2disconf". + #Include conf-available/serve-cgi-bin.conf + + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet +EOF + +cat >/var/www/html/index.html <<'EOF' + + + + + + + + + + +
+ + + +EOF + +# Copy Leaflet files to Apache's directory so they can be used offline. +cp /opt/wrolpi/modules/map/leaflet.js /var/www/html/leaflet.js +cp /opt/wrolpi/modules/map/leaflet.css /var/www/html/leaflet.css + +systemctl enable renderd +systemctl stop renderd +systemctl start renderd + +systemctl stop apache2 +systemctl start apache2 diff --git a/scripts/ubuntu_20.04_install_map.sh b/scripts/install_map_ubuntu_20.04.sh similarity index 100% rename from scripts/ubuntu_20.04_install_map.sh rename to scripts/install_map_ubuntu_20.04.sh diff --git a/scripts/ubuntu_20.04_install.sh b/scripts/install_ubuntu_20.04.sh similarity index 98% rename from scripts/ubuntu_20.04_install.sh rename to scripts/install_ubuntu_20.04.sh index 64540c9c..43089840 100755 --- a/scripts/ubuntu_20.04_install.sh +++ b/scripts/install_ubuntu_20.04.sh @@ -1,10 +1,9 @@ #!/bin/bash +echo "Ubuntu 20.04 install start: $(date '+%Y-%m-%d %H:%M:%S')" set -x set -e -echo "Ubuntu 20.04 install start: $(date '+%Y-%m-%d %H:%M:%S')" - # Install npm repos. curl -fsSL https://deb.nodesource.com/setup_14.x | bash - # Update if we haven't updated in the last day. @@ -78,7 +77,7 @@ sudo -u postgres psql -c '\l' | grep wrolpi || ( (cd /opt/wrolpi && /usr/bin/python3 /opt/wrolpi/main.py db upgrade) # Run the map installation script. -/opt/wrolpi/scripts/ubuntu_20.04_install_map.sh +/opt/wrolpi/scripts/install_map_ubuntu_20.04.sh set +x