diff --git a/README.md b/README.md index 8a33095..e9a0c70 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# kiwix-zim.sh +# kiwix-zim-updater.sh A script to check `download.kiwix.org` for updates to your local ZIM library. Just pass this script your ZIM directory and away it goes. *(see Usage below)* -Tested on PopOS! 22.04, and should work out of the box on most debian systems, but I have not tested that. +Tested on Debian 13, and should work out of the box on most debian systems, but I have not tested that. ## What It Does @@ -53,6 +53,7 @@ Creates `downloads.log` for the following reasons: ## Limitations +- This script relies heavily on `grep`'s `-P` flag for lookarounds and the matcher reset, `\K`. This may cause compatibility problems on macOS - This script is only for ZIM(s) hosted by `download.kiwix.org` due to the file naming standard they use. If you have self-made ZIM(s) or ZIM(s) downloaded from somewhere else, they most likely do not use the same naming standards and will not be processed by this script. - If you have ZIM(s) from `download.kiwix.org`, but you have changed their file names, this script will treat them like the previous limitation explains. - This script does not attempt to update any `library.xml` that may or may not exist/be needed for your install/setup of Kiwix. If needed, you'll need to handle this part on your own. @@ -69,8 +70,8 @@ Not checked or installed via script: This script is self-updating. The self-update routine uses git commands to make the update so this script should be "installed" with the below command. -```text -git clone https://github.com/DocDrydenn/kiwix-zim.git +```shell +git clone https://github.com/jojo2357/kiwix-zim-updater.git ``` UPDATE: If you decide not to install via a git clone, you can still use this script, however, it will just skip the update check and continue on. @@ -79,24 +80,31 @@ NOTE: if you are not tracking the `main` branch, the update check will be skippe ## Usage ```text - Usage: ./kiwix-zim.sh /full/path/ + Usage: ./kiwix-zim-updater.sh /full/path/ /full/path/ Full path to ZIM directory - Options: - -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads. - -f, --verify-library Verifies that the entire library has the correct checksums as found online. - Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet - -d, --disable-dry-run Dry-Run Override. - *** Caution *** - + Universal Options: -h, --help Show this usage and exit. - -p, --skip-purge Skips purging any replaced ZIMs. + -d, --disable-dry-run Dry-Run Override. + *** Caution *** -u, --skip-update Skips checking for script updates (very useful for development). + -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day -n , --min-size Minimum ZIM Size to be downloaded. Specify units include M Mi G Gi, etc. See `man numfmt` -x , --max-size Maximum ZIM Size to be downloaded. - Specify units include M Mi G Gi, etc. See `man numfmt` + Specify units include M Mi G Gi, etc. See `man numfmt` + + Action Method Options: + -w, --web Downloads zims over http(s). + -t, --torrent Downloads `.torrent` files. REQUIRES ADDITIONAL SOFTWARE TO EXECUTE DOWNLOAD. + + Web Download Options: + -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads. + -p, --skip-purge Skips purging any replaced ZIMs. -l , --location Country Code to prefer mirrors from - -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day + -f, --verify-library Verifies that the entire library has the correct checksums as found online. + Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet. + Disable this using -S + -S, --no-sha Disables saving the zim checksum for future reference. Does not delete present checksums. ``` diff --git a/kiwix-zim.sh b/kiwix-zim-updater.sh similarity index 50% rename from kiwix-zim.sh rename to kiwix-zim-updater.sh index ef130a9..7b5b692 100755 --- a/kiwix-zim.sh +++ b/kiwix-zim-updater.sh @@ -1,6 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash -VER="3.0" +# Kiwix-Zim-Updater. A script to update your kiwix library +# Copyright (C) 2022 DocDrydenn +# Copyright (C) 2023-2026 jojo2357 +# SPDX-License-Identifier: GPL-2.0-only + +VER="3.5" # This array will contain all of the local zims, with the file extension LocalZIMArray=() @@ -8,7 +13,7 @@ LocalZIMArray=() LocalZIMNameArray=() # This array will map the local zim to the index in the remote arrays that contains the same base file name LocalZIMRemoteIndexArray=() -# This array is a boolean array which remembers if a given local zim shoud be processed in the download loop +# This array is a boolean array which remembers if a given local zim should be processed in the download loop LocalRequiresDownloadArray=() # After updating, this array will be used to store hanging locks and to deal with them HangingFileLocks=(); @@ -26,7 +31,9 @@ RemoteCategory=() SCRIPT="$(readlink -f "$0")" SCRIPTFILE="$(basename "$SCRIPT")" SCRIPTPATH="$(dirname "$SCRIPT")" -SCRIPTNAME="$0" +CALLEDSCRIPTNAME="$0" +CALLEDSCRIPTFILE="$(basename "$CALLEDSCRIPTNAME")" +CALLEDSCRIPTPATH="$(dirname "$CALLEDSCRIPTNAME")" ARGS=("$@") BRANCH="main" SKIP_UPDATE=0 @@ -34,11 +41,31 @@ DEBUG=1 # This forces the script to default to "dry-run/simulation mode" MIN_SIZE=0 MAX_SIZE=0 CALCULATE_CHECKSUM=0 +CHECKSUM_FILES=1 VERIFY_LIBRARY=0 FORCE_FETCH_INDEX=0 -BaseURL="https://download.kiwix.org/zim/" +DOWNLOAD_METHOD=1 # 1: web 2: torrent +# sort of a default, will be overwritten when the library is parsed. Set here to what it probably will be +BaseURL="https://lb.download.kiwix.org/zim/" +CatalogURL="https://opds.library.kiwix.org/catalog/v2/entries?count=-1" ZIMPath="" +RED_REGULAR="\033[0;31m" +RED_BOLD="\033[1;31m" +YELLOW_REGULAR="\033[0;33m" +YELLOW_BOLD="\033[1;33m" +GREEN_REGULAR="\033[0;32m" +GREEN_BOLD="\033[1;32m" +BLUE_REGULAR="\033[0;34m" +BLUE_BOLD="\033[1;34m" +CLEAR="\033[0m" + +MONTH_REGEX="\d{2}[a-z]?" +YEAR_REGEX="\d{4}" +END_ANCHOR_REGEX="\.zim" + +COMPLETE_ENDING_REGEX="$YEAR_REGEX-$MONTH_REGEX$END_ANCHOR_REGEX" + # This will ask the api what files it has to offer and store them in arrays master_scrape() { unset RemoteFiles @@ -52,71 +79,116 @@ master_scrape() { indexIsValid=0 else indexDate="$(head -1 "kiwix-index")" - if [[ "$(date -u -d "$indexDate" +%s)" -lt "$(date -u -d "1 day ago" +%s)" ]]; then + if [[ -z "${indexDate}" ]] || [[ "$(date -u -d "$indexDate" +%s)" -lt "$(date -u -d "1 day ago" +%s)" ]]; then indexIsValid=0 fi fi if [[ FORCE_FETCH_INDEX -eq 1 ]] || [[ $indexIsValid -eq 0 ]]; then - # both write the file timestamp to the index file and save all of the links to RawLibrary - RawLibrary="$(wget --show-progress -q -O - "https://library.kiwix.org/catalog/v2/entries?count=-1" | tee --output-error=warn-nopipe >(grep -ioP "(?<=)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?=Z)" | head -1 > kiwix-index) | grep -i 'application/x-zim' | grep -ioP "^\s+\K.*$")" - + # both write the file timestamp to the index file and save all of the links to RawLibrary. + # Lets defer touching the index until we have actually fetched it + RawLibrary="$(wget --show-progress -q -O - "$CatalogURL")" + RawLibrary="$(echo "$RawLibrary" | tee --output-error=warn-nopipe >(grep -ioP "(?<=)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?=Z)" | head -1 > kiwix-index) | grep -i 'application/x-zim' | grep -ioP "^\s+\K.*$")" echo "$RawLibrary" >> kiwix-index else RawLibrary=$(grep -i '> download.log exit 0 else - echo -e "\033[1;32m ✓ Found ${#RemoteFiles[@]} files online" + echo -e "${GREEN_BOLD} ✓ Found ${#RemoteFiles[@]} files online" echo "✓ Found ${#RemoteFiles[@]} files online" >> download.log fi + if [[ ${#RemoteFiles[@]} -ne ${#RemoteFiles[@]} ]] || \ + [[ ${#RemoteFiles[@]} -ne ${#Basenames[@]} ]] || \ + [[ ${#RemoteFiles[@]} -ne ${#RemotePaths[@]} ]] || \ + [[ ${#RemoteFiles[@]} -ne ${#RemoteCategory[@]} ]] || \ + [[ ${#RemoteFiles[@]} -ne ${#FileSizes[@]} ]] ; then + echo -e "${RED_BOLD}CRITICAL ERROR!${CLEAR}" + echo -e "${RED}Parsed array lengths did not match. Aborting to prevent data loss${CLEAR}" + exit 99 + fi + # Housekeeping... + unset ValidLibrary unset RawLibrary unset hrefs } # self_update - Script Self-Update Function self_update() { - echo -e "\033[1;33m1. Checking for Script Updates...\033[0m" + echo -e "${YELLOW_BOLD}1. Checking for Script Updates...${CLEAR}" echo "1. Checking for Script Updates..." >> download.log echo # Check if script path is a git clone. # If true, then check for update. - # If false, skip self-update check/funciton. + # If false, skip self-update check/function. if [ $SKIP_UPDATE -eq 1 ]; then - echo -e "\033[0;33m Check Skipped\033[0m" + echo -e "${YELLOW_REGULAR} Check Skipped${CLEAR}" echo "Check Skipped" >> download.log elif [[ -d "$SCRIPTPATH/.git" ]]; then - echo -e "\033[1;32m ✓ Git Clone Detected: Checking Script Version...\033[0m" + echo -e "${GREEN_BOLD} ✓ Git Clone Detected: Checking Script Version...${CLEAR}" echo "✓ Git Clone Detected: Checking Script Version..." >> download.log cd "$SCRIPTPATH" || exit 1 - [[ $(timeout 1s git rev-parse --abbrev-ref HEAD) != "$BRANCH" ]] && echo -e "\033[1;33m You appear to be on a different branch so I will assume you are developing and do not want an update\033[0m" && echo && return + [[ $(timeout 1s git rev-parse --abbrev-ref HEAD) != "$BRANCH" ]] && echo -e "${YELLOW_BOLD} You appear to be on a different branch so I will assume you are developing and do not want an update${CLEAR}" && echo && return timeout 1s git fetch --quiet timeout 1s git diff --quiet --exit-code "origin/$BRANCH" "$SCRIPTFILE" [ $? -eq 1 ] && { - echo -e "\033[0;31m ✗ Version: Mismatched\033[0m" + echo -e "${RED_REGULAR} ✗ Version: Mismatched${CLEAR}" echo "✗ Version: Mismatched" >> download.log echo - echo -e "\033[1;33m1a. Fetching Update...\033[0m" + echo -e "${YELLOW_BOLD}1a. Fetching Update...${CLEAR}" echo "1a. Fetching Update..." >> download.log echo if [ -n "$(git status --porcelain)" ]; then @@ -125,56 +197,64 @@ self_update() { git pull --force --quiet git checkout $BRANCH --quiet git pull --force --quiet - echo -e "\033[1;32m ✓ Update Complete. Running New Version. Standby...\033[0m" + echo -e "${GREEN_BOLD} ✓ Update Complete. Running New Version. Standby...${CLEAR}" echo "✓ Update Complete. Running New Version. Standby..." >> download.log sleep 3 cd - >/dev/null || exit 1 - # Execute new instance of the new script - exec "$SCRIPTNAME" "${ARGS[@]}" + exec "$CALLEDSCRIPTNAME" "${ARGS[@]}" # Exit this old instance of the script exit 1 } - echo -e "\033[1;32m ✓ Version: Current\033[0m" + echo -e "${GREEN_BOLD} ✓ Version: Current${CLEAR}" else - echo -e "\033[0;31m ✗ Git Clone Not Detected: Skipping Update Check\033[0m" + echo -e "${RED_REGULAR} ✗ Git Clone Not Detected: Skipping Update Check${CLEAR}" fi echo } # usage_example - Show Usage and Exit usage_example() { - echo 'Usage: ./kiwix-zim.sh /full/path/' - echo + echo 'Usage: ./kiwix-zim-updater.sh /full/path/' + echo '' echo ' /full/path/ Full path to ZIM directory' - echo - echo 'Options:' - echo ' -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads.' - echo ' -f, --verify-library Verifies that the entire library has the correct checksums as found online.' - echo ' Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet' - echo ' -d, --disable-dry-run Dry-Run Override.' - echo ' *** Caution ***' - echo + echo '' + echo 'Universal Options:' echo ' -h, --help Show this usage and exit.' - echo ' -p, --skip-purge Skips purging any replaced ZIMs.' echo ' -u, --skip-update Skips checking for script updates (very useful for development).' + echo ' -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day' echo ' -n , --min-size Minimum ZIM Size to be downloaded.' echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' echo ' -x , --max-size Maximum ZIM Size to be downloaded.' echo ' Specify units include M Mi G Gi, etc. See `man numfmt`' + echo ' -S, --no-sha Disables saving the zim checksum for future reference. Does not delete present checksums.' + echo ' ' + echo 'Action Method Options:' + echo ' -w, --web Downloads zims over http(s).' + echo ' -t, --torrent Downloads `.torrent` files. REQUIRES ADDITIONAL SOFTWARE TO EXECUTE DOWNLOAD.' + echo ' ' + echo ' -f, --verify-library Verifies that the entire library has the correct checksums as found online.' + echo ' Expected behavior is to create sha256 files during a normal run so this option can be used at a later date without internet.' + echo ' Disable this using -S' + echo '' + echo 'Web Download Options:' + echo ' -c, --calculate-checksum Verifies that the downloaded files were not corrupted, but can take a while for large downloads.' + echo ' -p, --skip-purge Skips purging any replaced ZIMs.' echo ' -l , --location Country Code to prefer mirrors from' - echo ' -g, --get-index Forces using remote index rather than cached index. Cache auto clears after one day' + echo ' -d, --disable-dry-run Dry-Run Override.' + echo ' *** Caution ***' + echo exit 0 } # flags - Flag and ZIM Processing Functions flags() { - echo -e "\033[1;33m2. Preprocessing...\033[0m" + echo -e "${YELLOW_BOLD}2. Preprocessing...${CLEAR}" echo "2. Preprocessing..." >> download.log echo - echo -e "\033[1;34m -Validating ZIM directory...\033[0m" + echo -e "${BLUE_BOLD} -Validating ZIM directory...${CLEAR}" echo "Validating ZIM directory..." >> download.log # Let's identify which argument is the ZIM directory path and if it's an actual directory. @@ -183,17 +263,17 @@ flags() { ZIMPath=$1 else ZIMPath=$1 - echo -e "\033[0;31m ✗ Cannot write to '${1}', continuing in dry-run\033[0m" + echo -e "${RED_REGULAR} ✗ Cannot write to '${1}', continuing in dry-run${CLEAR}" echo "✗ Cannot write to '${1}', continuing in dry-run" >> download.log echo DEBUG=1 fi else # Um... no ZIM directory path provided? Okay, let's show the usage and exit. if [[ -z ${1} ]]; then - echo -e "\033[0;31m ✗ Kiwix ZIM Directory not provided\033[0m" + echo -e "${RED_REGULAR} ✗ Kiwix ZIM Directory not provided${CLEAR}" echo "✗ Kiwix ZIM Directory not provided" >> download.log else - echo -e "\033[0;31m ✗ '$1' is not a directory\033[0m" + echo -e "${RED_REGULAR} ✗ '$1' is not a directory${CLEAR}" echo "✗ '$1' is not a directory" >> download.log fi echo @@ -212,19 +292,43 @@ flags() { for index in "${!LocalZIMArray[@]}" ; do duplicated=0 - myBasename=$(echo "${LocalZIMArray[$index]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") + myBasename=$(echo "${LocalZIMArray[$index]}" | grep -ioP "^.*(?=_$COMPLETE_ENDING_REGEX$)") + + # Sorting cannot reliably sort the month string so we need to check it to be sure + MyDate="$(echo "${LocalZIMArray[$index]}" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" + MyYear="$(echo "$MyDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" + MyMonth="$(echo "$MyDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" + for scanIndex in "${!LocalZIMArray[@]}"; do if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then if [[ $index -le $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi else if [[ $index -ge $scanIndex ]] || [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$scanIndex]}" ]]; then continue; fi fi - scanBasename=$(echo "${LocalZIMArray[$scanIndex]}" | grep -ioP "^.*(?=_\d{4}-\d{2}\.zim$)") + scanBasename=$(echo "${LocalZIMArray[$scanIndex]}" | grep -ioP "^.*(?=_$COMPLETE_ENDING_REGEX$)") if [[ "$myBasename" == "$scanBasename" ]]; then if [[ -f "${ZIMPath}.~lock.${LocalZIMArray[$index]}" ]]; then echo "Disregarding ${LocalZIMArray[$index]} because it was interrupted by ${LocalZIMArray[$scanIndex]}" >> download.log + elif [[ -f "${ZIMPath}${LocalZIMArray[$index]}.torrent" ]]; then + echo "Disregarding ${LocalZIMArray[$index]} because new torrent exists ${LocalZIMArray[$scanIndex]}" >> download.log else + ScanDate="$(echo "${LocalZIMArray[$scanIndex]}" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" + ScanYear="$(echo "$ScanDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" + ScanMonth="$(echo "$ScanDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" + + # this file is not shadowed if it is newer. + # this is caused by `ls -1` and `sort` having weird lexicons that make the prefix sort unreliable that 02a can sort between 01 and 02 instead of after 02 + if [[ $MyYear > $ScanYear ]]; then + echo "Disregarding ${LocalZIMArray[$scanIndex]} because it is shadowed by ${LocalZIMArray[$index]}" >> download.log + duplicated=2 + break + elif [[ $MyYear == $ScanYear ]] && [[ $MyMonth > $ScanMonth ]]; then + echo "Disregarding ${LocalZIMArray[$scanIndex]} because it is shadowed by ${LocalZIMArray[$index]}" >> download.log + duplicated=2 + break + fi + echo "Disregarding ${LocalZIMArray[$index]} because it is shadowed by ${LocalZIMArray[$scanIndex]}" >> download.log fi duplicated=1 @@ -232,19 +336,20 @@ flags() { fi done [[ $duplicated -eq 1 ]] && unset -v 'LocalZIMArray[$index]' + [[ $duplicated -eq 2 ]] && unset -v 'LocalZIMArray[$scanIndex]' done LocalZIMArray=("${LocalZIMArray[@]}") # Check that ZIM(s) were actually found/loaded. if [ ${#LocalZIMArray[@]} -eq 0 ]; then # No ZIM(s) were found in the directory... I guess there's nothing else for us to do, so we'll Exit. - echo -e "\033[0;31m ✗ No ZIMs found. Exiting...\033[0m" + echo -e "${RED_REGULAR} ✗ No ZIMs found. Exiting...${CLEAR}" exit 0 else - echo -e "\033[1;32m ✓ Valid ZIM Directory \033[0m" + echo -e "${GREEN_BOLD} ✓ Valid ZIM Directory ${CLEAR}" fi echo - echo -e "\033[1;34m -Building online ZIM list...\033[0m" + echo -e "${BLUE_BOLD} -Building online ZIM list...${CLEAR}" # Build online ZIM list. master_scrape @@ -252,11 +357,11 @@ flags() { echo # Populate ZIM arrays from found ZIM(s) - echo -e "\033[1;34m -Parsing ZIM(s)...\033[0m" + echo -e "${BLUE_BOLD} -Parsing ZIM(s)...${CLEAR}" for ((i = 0; i < ${#LocalZIMArray[@]}; i++)); do # Loop through local ZIM(s). LocalZIMNameArray[$i]=$(basename "${LocalZIMArray[$i]}") # Extract file name. - filename=$(basename "${LocalZIMArray[$i]}" | grep -ioP "[\w:\/\-.]+(?=\d{4}-\d{2}\.zim$)") # Extract file name. + filename=$(basename "${LocalZIMArray[$i]}" | grep -ioP "[\w:\/\-.]+(?=$COMPLETE_ENDING_REGEX$)") # Extract file name. # IFS='_' read -ra fields <<< "${LocalZIMNameArray[$i]}"; unset IFS # Break the filename into fields delimited by the underscore '_' # Search MasterZIMArray for the current local ZIM to discover the online Root (directory) for the URL @@ -270,14 +375,14 @@ flags() { done if [[ LocalZIMRemoteIndexArray[$i] -eq -1 ]]; then - echo -e "\033[0;31m ✗ ${LocalZIMNameArray[$i]} No online match found.\033[0m" + echo -e "${RED_REGULAR} ✗ ${LocalZIMNameArray[$i]} No online match found.${CLEAR}" else - echo -e "\033[1;32m ✓ ${LocalZIMNameArray[$i]} [${RemoteCategory[${LocalZIMRemoteIndexArray[$i]}]}]\033[0m" + echo -e "${GREEN_BOLD} ✓ ${LocalZIMNameArray[$i]} [${RemoteCategory[${LocalZIMRemoteIndexArray[$i]}]}]${CLEAR}" fi done echo - echo -e "\033[0;32m ${#LocalZIMNameArray[*]} ZIM(s) found.\033[0m" + echo -e "${GREEN_REGULAR} ${#LocalZIMNameArray[*]} ZIM(s) found.${CLEAR}" } # mirror_search - Find ZIM URL Priority #1 mirror from meta4 Function @@ -292,6 +397,13 @@ mirror_search() { MetaInfo=$(wget -q -O - "$BaseURL$RemotePath.meta4?country=$COUNTRY_CODE") ExpectedSize=$(echo "$MetaInfo" | grep '' | grep -Po '\d+') ExpectedHash=$(echo "$MetaInfo" | grep '' | grep -Poi '(?<="sha-256">)[a-f\d]{64}(?=<)') + + # still run the last wget so that the logic is simple for verification. + if [[ $DOWNLOAD_METHOD -eq 2 ]]; then + DownloadURL="$BaseURL$RemotePath.torrent" # Set the direct download URL as our download URL + return + fi + RawMirror=$(echo "$MetaInfo" | grep 'priority="1"' | grep -Po 'https?://[^ ")]+(?=)') # Check that we actually got a URL (this could probably be done better). If no mirror URL, default back to direct URL. @@ -340,7 +452,7 @@ while [[ $# -gt 0 ]]; do COUNTRY_CODE=$1 # convert passed arg to bytes else COUNTRY_CODE="" - echo "Invlaid country code, falling back to default kiwix behavior" + echo "Invalid country code, falling back to default kiwix behavior" >> download.log fi shift # discard value ;; @@ -348,7 +460,7 @@ while [[ $# -gt 0 ]]; do CALCULATE_CHECKSUM=1 shift ;; - -f | --verfiy-library) + -f | --verfiy-library | --verify-library) VERIFY_LIBRARY=1 CALCULATE_CHECKSUM=1 shift @@ -361,6 +473,18 @@ while [[ $# -gt 0 ]]; do FORCE_FETCH_INDEX=1 shift ;; + -w | --web) + DOWNLOAD_METHOD=1 + shift + ;; + -t | --torrent) + DOWNLOAD_METHOD=2 + shift + ;; + -S | --no-sha) + CHECKSUM_FILES=0 + shift + ;; *) # We can either parse the arg here, or just tuck it away for safekeeping POSITIONAL_ARGS+=("$1") # save positional arg @@ -373,9 +497,18 @@ set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters that we skipped e clear # Clear screen +if [[ $CALCULATE_CHECKSUM -ne 0 ]] && [[ $DOWNLOAD_METHOD -ne 1 ]]; then + echo -e "${RED_BOLD}Calculating Checksum not available with torrenting. Aborting.${CLEAR}" +fi + # Display Header echo "==========================================" -echo " kiwix-zim" +echo " kiwix-zim-updater" +if [[ "$CALLEDSCRIPTFILE" == "kiwix-zim.sh" ]]; then + echo " WARNING: kiwix-zim.sh has been deprecated " + echo " in favor of kiwix-zim-updater " + echo "WARNING: kiwix-zim has been deprecated in favor of kiwix-zim-updater. kiwix-zim will be a hard link to the new script name until at least 2025-01-01" >> download.log +fi echo " download.kiwix.org ZIM Updater" echo echo " v$VER by DocDrydenn and jojo2357" @@ -400,7 +533,7 @@ self_update flags "$@" echo -echo -e "\033[1;33m3. Processing ZIM(s)...\033[0m" +echo -e "${YELLOW_BOLD}3. Processing ZIM(s)...${CLEAR}" echo "3. Processing ZIM(s)..." >> download.log echo @@ -412,8 +545,8 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do if [[ $VERIFY_LIBRARY -eq 1 ]] && [[ -f "$FileName.sha256" ]]; then LocalRequiresDownloadArray+=(3) AnyDownloads=1 - echo -e "\033[1;34m - $FileName:\033[0m" - echo -e "\033[0;32m Cached Checksum Found\033[0m" + echo -e "${BLUE_BOLD} - $FileName:${CLEAR}" + echo -e "${GREEN_REGULAR} Cached Checksum Found${CLEAR}" else LocalRequiresDownloadArray+=(0) fi @@ -421,14 +554,25 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do fi FileName=${LocalZIMNameArray[$i]} - echo -e "\033[1;34m - $FileName:\033[0m" - [[ -f "$ZIMPath.~lock.$FileName" ]] && echo -e "\033[0;33m Incomplete download detected\n\033[1;32m ✓ Online Version Found\033[0m\n" && LocalRequiresDownloadArray+=(1) && AnyDownloads=1 && continue + echo -e "${BLUE_BOLD} - $FileName:${CLEAR}" + [[ -f "$ZIMPath.~lock.$FileName" ]] && echo -e "${YELLOW_REGULAR} Incomplete download detected\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(1) && AnyDownloads=1 && continue + MatchingSize=${FileSizes[$RemoteIndex]} MatchingFileName=${RemoteFiles[$RemoteIndex]} MatchingFullPath=${RemotePaths[$RemoteIndex]} MatchingCategory=${RemoteCategory[$RemoteIndex]} + [[ -f "$ZIMPath$MatchingFileName.torrent" ]] && [[ ! -f "$ZIMPath$MatchingFileName" ]] && echo -e "${YELLOW_REGULAR} Torrent already downloaded\n${GREEN_BOLD} ✓ Online Version Found${CLEAR}\n" && LocalRequiresDownloadArray+=(0) && continue + + MatchedDate="$(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" + MatchedYear="$(echo "$MatchedDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" + MatchedMonth="$(echo "$MatchedDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" + + LocalDate="$(echo "$FileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")" + LocalYear="$(echo "$LocalDate" | grep -oP "$YEAR_REGEX(?=-$MONTH_REGEX)")" + LocalMonth="$(echo "$LocalDate" | grep -oP "(?<=$YEAR_REGEX-)$MONTH_REGEX")" + FileTooSmall=0 [[ $MIN_SIZE -gt 0 ]] && [[ $MatchingSize -lt $MIN_SIZE ]] && FileTooSmall=1 FileTooLarge=0 @@ -439,18 +583,18 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do if [ $VERIFY_LIBRARY -eq 1 ] && [ $FileSizeAcceptable -eq 0 ]; then if [ $FileTooSmall -eq 1 ]; then LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "\033[0;32m ✓ Verification skipped due to file size (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))\033[0m" - [[ $DEBUG -eq 1 ]] && echo -e "\033[0;32m ✓ *** Simulated *** Verification skipped due to file size (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))\033[0m" + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Verification skipped due to file size (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Verification skipped due to file size (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" elif [ $FileTooLarge -eq 1 ]; then LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "\033[0;32m ✓ Verification skipped due to file size (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))\033[0m" - [[ $DEBUG -eq 1 ]] && echo -e "\033[0;32m ✓ *** Simulated *** Verification skipped due to file size (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))\033[0m" + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Verification skipped due to file size (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Verification skipped due to file size (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize"))${CLEAR}" fi elif [[ "$MatchingFileName" == "$FileName" ]]; then if [ $VERIFY_LIBRARY -eq 1 ]; then LocalRequiresDownloadArray+=(1) AnyDownloads=1 - echo -e "\033[1;32m ✓ Online Version Found\033[0m" + echo -e "${GREEN_BOLD} ✓ Online Version Found${CLEAR}" else LocalRequiresDownloadArray+=(0) echo " ✗ No new update" @@ -460,13 +604,13 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do if [[ -f "$ZIMPath$FileName.sha256" ]]; then LocalRequiresDownloadArray+=(2) AnyDownloads=1 - echo -e "\033[0;32m Cached Checksum Found\033[0m" + echo -e "${GREEN_REGULAR} Cached Checksum Found${CLEAR}" else echo " Checking for online checksum..." if wget -S --spider -q -O - "$BaseURL$MatchingCategory/$FileName.sha256" >/dev/null 2>&1; then LocalRequiresDownloadArray+=(1) AnyDownloads=1 - echo -e "\033[1;32m ✓ Online Version Found\033[0m" + echo -e "${GREEN_BOLD} ✓ Online Version Found${CLEAR}" else LocalRequiresDownloadArray+=(0) echo " ✗ Online Version Not Found" @@ -475,23 +619,29 @@ for ((i = 0; i < ${#LocalZIMNameArray[@]}; i++)); do else if [ $FileTooSmall -eq 1 ]; then LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "\033[0;32m ✓ Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')\033[0m" - [[ $DEBUG -eq 1 ]] && echo -e "\033[0;32m ✓ *** Simulated *** Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')\033[0m" + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (minimum: $(numfmt --to=iec-i $MIN_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" elif [ $FileTooLarge -eq 1 ]; then LocalRequiresDownloadArray+=(0) - [[ $DEBUG -eq 0 ]] && echo -e "\033[0;32m ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')\033[0m" - [[ $DEBUG -eq 1 ]] && echo -e "\033[0;32m ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')\033[0m" + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ *** Simulated *** Update skipped (maximum: $(numfmt --to=iec-i $MAX_SIZE), download size: $(numfmt --to=iec-i "$MatchingSize")). New version: $(echo "$MatchingFileName" | grep -oP "$YEAR_REGEX-$MONTH_REGEX(?=$END_ANCHOR_REGEX$)")${CLEAR}" + elif [[ $MatchedYear < $LocalYear ]]; then + LocalRequiresDownloadArray+=(0) + echo " ✗ No new update" + elif [[ "$MatchedYear" == "$LocalYear" ]] && { [[ $MatchedMonth < $LocalMonth ]] || [[ "$MatchedMonth" == "$LocalMonth" ]] ; }; then + LocalRequiresDownloadArray+=(0) + echo " ✗ No new update" else LocalRequiresDownloadArray+=(1) AnyDownloads=1 - echo -e "\033[1;32m ✓ Update found! --> $(echo "$MatchingFileName" | grep -oP '\d{4}-\d{2}(?=\.zim$)')\033[0m" + echo -e "${GREEN_BOLD} ✓ Update found! --> $MatchedDate${CLEAR}" fi fi fi echo done -echo -e "\033[1;33m4. Downloading New ZIM(s)...\033[0m" +echo -e "${YELLOW_BOLD}4. Downloading New ZIM(s)...${CLEAR}" echo -e "4. Downloading New ZIM(s)..." >> download.log echo @@ -505,7 +655,7 @@ if [ $AnyDownloads -eq 1 ]; then OldZIM=${LocalZIMNameArray[$z]} OldZIMPath=$ZIMPath$OldZIM - echo -e "\033[1;34m Processing $OldZIM\033[0m" + echo -e "${BLUE_BOLD} Processing $OldZIM${CLEAR}" if [[ ${LocalRequiresDownloadArray[$z]} -eq 3 ]]; then ExpectedHash=$(grep -ioP "^[0-9a-f]{64}" <"$OldZIMPath.sha256") @@ -523,24 +673,40 @@ if [ $AnyDownloads -eq 1 ]; then NewZIMPath=$ZIMPath$NewZIM fi + FilePath=$ZIMPath$NewZIM # Set destination path with file name LockFilePath="$ZIMPath.~lock.$NewZIM" # Set destination path with file name + TorrentFilePath="$ZIMPath$NewZIM.torrent" # Set destination path with file name RequiresDownload=0 if [ $VERIFY_LIBRARY -eq 0 ]; then - if [[ -f $NewZIM ]] && ! [[ -f $LockFilePath ]]; then # New ZIM already found, and no interruptions, we don't need to download it. - [[ $DEBUG -eq 0 ]] && echo -e "\033[0;32m ✓ Status : ZIM already exists on disk. Skipping download.\033[0m" - [[ $DEBUG -eq 1 ]] && echo -e "\033[0;32m ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.\033[0m" + if [[ $DOWNLOAD_METHOD -eq 2 ]]; then + if [[ -f $NewZIM ]]; then + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" + echo + elif [[ -f $TorrentFilePath ]]; then + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent already exists on disk. Skipping download.${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent already exists on disk. Skipping download.${CLEAR}" + echo + else + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : Torrent doesn't exist on disk. Downloading...${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** Torrent doesn't exist on disk. Downloading...${CLEAR}" + RequiresDownload=1 + fi + elif [[ -f $NewZIM ]] && ! [[ -f $LockFilePath ]]; then # New ZIM already found, and no interruptions, we don't need to download it. + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM already exists on disk. Skipping download.${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM already exists on disk. Skipping download.${CLEAR}" echo else # New ZIM not found, so we'll go ahead and download it. RequiresDownload=1 if [[ -f $LockFilePath ]]; then - [[ $DEBUG -eq 0 ]] && echo -e "\033[0;32m ✓ Status : ZIM download was interrupted. Continuing...\033[0m" - [[ $DEBUG -eq 1 ]] && echo -e "\033[0;32m ✓ Status : *** Simulated *** ZIM download was interrupted. Continuing...\033[0m" + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM download was interrupted. Continuing...${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM download was interrupted. Continuing...${CLEAR}" else - [[ $DEBUG -eq 0 ]] && echo -e "\033[0;32m ✓ Status : ZIM doesn't exist on disk. Downloading...\033[0m" - [[ $DEBUG -eq 1 ]] && echo -e "\033[0;32m ✓ Status : *** Simulated *** ZIM doesn't exist on disk. Downloading...\033[0m" + [[ $DEBUG -eq 0 ]] && echo -e "${GREEN_REGULAR} ✓ Status : ZIM doesn't exist on disk. Downloading...${CLEAR}" + [[ $DEBUG -eq 1 ]] && echo -e "${GREEN_REGULAR} ✓ Status : *** Simulated *** ZIM doesn't exist on disk. Downloading...${CLEAR}" fi echo fi @@ -548,30 +714,33 @@ if [ $AnyDownloads -eq 1 ]; then # lockfile implies an incomplete download if [[ -f $LockFilePath ]]; then echo "Incomplete download detected, resuming" >>download.log - echo -e "\033[0;33m Status: Incomplete download detected, resuming\033[0m" + echo -e "${YELLOW_REGULAR} Status: Incomplete download detected, resuming${CLEAR}" - # [[ $IsMirror -eq 0 ]] && echo -e "\033[1;34m Download (direct) : $DownloadURL\033[0m" - # [[ $IsMirror -eq 1 ]] && echo -e "\033[1;34m Download (mirror) : $DownloadURL\033[0m" + # [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_BOLD} Download (direct) : $DownloadURL${CLEAR}" + # [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_BOLD} Download (mirror) : $DownloadURL${CLEAR}" RequiresDownload=1 + elif [[ -f $TorrentFilePath ]]; then + echo -e "${YELLOW_REGULAR} Status: Torrent exists, skipping${CLEAR}" + echo "Skipping checksum for torrent : $OldZIM. If the download has been completed, please delete the .torrent if you want me to verify the checksum." >>download.log else # actually verify the file - echo -e "\033[0;34m Calculating checksum for : $OldZIM\033[0m" + echo -e "${BLUE_REGULAR} Calculating checksum for : $OldZIM${CLEAR}" echo "Calculating checksum for : $OldZIM" >>download.log [[ ${LocalRequiresDownloadArray[$z]} -ne 2 ]] && echo "$ExpectedHash $OldZIM" >"$OldZIMPath.sha256" if [[ ${LocalRequiresDownloadArray[$z]} -ne 2 ]] && [[ $(du -b "$OldZIMPath" | grep -ioP "^\d+") -ne "$ExpectedSize" ]]; then RequiresDownload=1 if [[ $DEBUG -eq 0 ]]; then if [[ $SKIP_PURGE -eq 0 ]]; then - echo -e "\033[1;31m ✗ Status : File size verification failed, removing corrupt file\033[0m" + echo -e "${RED_BOLD} ✗ Status : File size verification failed, removing corrupt file${CLEAR}" echo "✗ Status : File size verification failed, removing corrupt file" >>download.log rm "$OldZIMPath" else - echo -e "\033[1;31m ✗ Status : File size verification failed but purge skipped\033[0m" + echo -e "${RED_BOLD} ✗ Status : File size verification failed but purge skipped${CLEAR}" echo "✗ Status : File size verification failed but purge skipped" >>download.log fi else - [[ $SKIP_PURGE -eq 0 ]] && echo -e "\033[1;31m ✗ Status : *** Simulated *** File size verification failed, removing corrupt file ($FilePath)\033[0m" - [[ $SKIP_PURGE -eq 1 ]] && echo -e "\033[1;31m ✗ Status : *** Simulated *** File size verification failed but purge skipped ($FilePath)\033[0m" + [[ $SKIP_PURGE -eq 0 ]] && echo -e "${RED_BOLD} ✗ Status : *** Simulated *** File size verification failed, removing corrupt file ($FilePath)${CLEAR}" + [[ $SKIP_PURGE -eq 1 ]] && echo -e "${RED_BOLD} ✗ Status : *** Simulated *** File size verification failed but purge skipped ($FilePath)${CLEAR}" fi echo @@ -579,27 +748,27 @@ if [ $AnyDownloads -eq 1 ]; then echo " This hash doesn't look quite right...skipping" elif (cd "$ZIMPath" && ! sha256sum --status -c "$OldZIM.sha256"); then # we checked a very old file, but we will choose to not replace it because we cannot, so we will leave it. A regular update will purge it naturally - [[ ${LocalRequiresDownloadArray[$z]} -eq 2 ]] && echo -e "\033[1;31m Checksum failed, online file not found, continuing\033[0m" && echo && continue + [[ ${LocalRequiresDownloadArray[$z]} -eq 2 ]] && echo -e "${RED_BOLD} Checksum failed, online file not found, continuing${CLEAR}" && echo && continue if [[ $DEBUG -eq 0 ]]; then if [[ $SKIP_PURGE -eq 0 ]]; then - echo -e "\033[1;31m ✗ Status : Checksum failed, removing corrupt file\033[0m" + echo -e "${RED_BOLD} ✗ Status : Checksum failed, removing corrupt file${CLEAR}" echo "✗ Status : Checksum failed, removing corrupt file" >>download.log rm "$OldZIMPath" rm "$OldZIMPath.sha256" 2>/dev/null else - echo -e "\033[1;31m ✗ Status : Checksum failed but purge was skipped\033[0m" + echo -e "${RED_BOLD} ✗ Status : Checksum failed but purge was skipped${CLEAR}" echo "✗ Status : Checksum failed but purge was skipped" >>download.log echo continue fi else - echo -e "\033[1;31m ✗ Status : *** Simulated *** Checksum failed, removing corrupt file ($FilePath)\033[0m" + echo -e "${RED_BOLD} ✗ Status : *** Simulated *** Checksum failed, removing corrupt file ($FilePath)${CLEAR}" fi RequiresDownload=1 echo else - echo -e "\033[1;32m ✓ Status : Checksum passed\033[0m" + echo -e "${GREEN_BOLD} ✓ Status : Checksum passed${CLEAR}" echo "✓ Status : Checksum passed" >>download.log [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log @@ -619,8 +788,9 @@ if [ $AnyDownloads -eq 1 ]; then [[ $DEBUG -eq 1 ]] && echo "Start : $(date -u) *** Simulation ***" >>download.log # Here is where we actually download the files and log to the download.log file. if [[ $RequiresDownload -eq 1 ]]; then - [[ $IsMirror -eq 0 ]] && echo -e "\033[0;34m Download (direct) : $DownloadURL\033[0m" - [[ $IsMirror -eq 1 ]] && echo -e "\033[0;34m Download (mirror) : $DownloadURL\033[0m" + [[ $IsMirror -eq 0 ]] && echo -e "${BLUE_REGULAR} Download (direct) : $DownloadURL${CLEAR}" + [[ $IsMirror -eq 1 ]] && echo -e "${BLUE_REGULAR} Download (mirror) : $DownloadURL${CLEAR}" + echo >>download.log echo "=======================================================================" >>download.log echo "File : $NewZIM" >>download.log @@ -628,43 +798,57 @@ if [ $AnyDownloads -eq 1 ]; then [[ $IsMirror -eq 1 ]] && echo "URL (mirror) : $DownloadURL" >>download.log echo >>download.log + if [[ $DOWNLOAD_METHOD -eq 2 ]]; then + FilePath="$FilePath.torrent" + if [[ -f "$LockFilePath" ]]; then + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log + elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. + [[ $DEBUG -eq 1 ]] && echo "Download : New Torrent already exists on disk. Skipping download." >>download.log + else # New ZIM not found, so we'll go ahead and download it. + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log + fi - # Before we actually download, let's just check to see that it isn't already in the folder. - if [[ -f "$LockFilePath" ]]; then - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log - elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. - [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log - else # New ZIM not found, so we'll go ahead and download it. - [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" - [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 |& tee -a download.log # Download new ZIM - [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log + continue + else + # Before we actually download, let's just check to see that it isn't already in the folder. + if [[ -f "$LockFilePath" ]]; then + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Continue Download : $FilePath" >>download.log + elif [[ -f $FilePath ]]; then # New ZIM already found, we don't need to download it. + [[ $DEBUG -eq 1 ]] && echo "Download : New ZIM already exists on disk. Skipping download." >>download.log + else # New ZIM not found, so we'll go ahead and download it. + [[ $DEBUG -eq 0 ]] && touch "$LockFilePath" + [[ $DEBUG -eq 0 ]] && wget -q --show-progress --progress=bar:force -c -O "$FilePath" "$DownloadURL" 2>&1 | tee -a download.log # Download new ZIM + [[ $DEBUG -eq 1 ]] && echo "Download : $FilePath" >>download.log + fi fi fi - echo "$ExpectedHash $NewZIM" 2>/dev/null 1>"$NewZIMPath.sha256" + [[ $DEBUG -eq 0 ]] && echo "$ExpectedHash $NewZIM" 2>/dev/null 1>"$NewZIMPath.sha256" if [[ $CALCULATE_CHECKSUM -eq 1 ]]; then - echo -e "\033[0;34m Calculating checksum for : $NewZIMPath\033[0m" + echo -e "${BLUE_REGULAR} Calculating checksum for : $NewZIMPath${CLEAR}" if [[ $(du -b "$NewZIMPath" 2>/dev/null | grep -ioP "^\d+") -ne "$ExpectedSize" ]]; then if [[ $DEBUG -eq 0 ]]; then - echo -e "\033[1;31m ✗ Status : File size verification failed, removing corrupt file\033[0m" + echo -e "${RED_BOLD} ✗ Status : File size verification failed, removing corrupt file${CLEAR}" echo "✗ Status : File size verification failed, removing corrupt file" >>download.log rm "$NewZIMPath" else - echo -e "\033[1;32m ✓ *** Simulated *** Checksum passed\033[0m" + echo -e "${GREEN_BOLD} ✓ *** Simulated *** Checksum passed${CLEAR}" fi elif [[ ${#ExpectedHash} -ne 64 ]]; then - echo -e "\033[1;33m This hash doesn't look quite right...skipping\033[0m" + echo -e "${YELLOW_BOLD} This hash doesn't look quite right...skipping${CLEAR}" elif [[ $DEBUG -eq 0 ]] && (cd "$ZIMPath" && ! sha256sum --status -c "$NewZIM.sha256"); then - echo -e "\033[1;31m ✗ Checksum failed, removing corrupt file\033[0m" + echo -e "${RED_BOLD} ✗ Checksum failed, removing corrupt file${CLEAR}" rm "$NewZIMPath" touch "$NewZIMPath" DownloadFailed=1 else if [[ $DEBUG -eq 0 ]]; then - echo -e "\033[1;32m ✓ Checksum passed\033[0m" + echo -e "${GREEN_BOLD} ✓ Checksum passed${CLEAR}" else - echo -e "\033[1;32m ✓ *** Simulated *** Checksum passed\033[0m" + echo -e "${GREEN_BOLD} ✓ *** Simulated *** Checksum passed${CLEAR}" fi fi # rm "$NewZIMPath.sha256" @@ -675,11 +859,12 @@ if [ $AnyDownloads -eq 1 ]; then echo >> download.log [[ $DownloadFailed -eq 1 ]] && echo " !!! DOWNLOAD FAILED !!!" >>download.log - # in all of these cases, we will not re-pruge and will leave the lockfile so we know to resume later + # in all of these cases, we will not re-purge and will leave the lockfile so we know to resume later if [[ $DownloadFailed -eq 1 ]] || [[ $SKIP_PURGE -eq 1 ]] || [[ $VERIFY_LIBRARY -eq 1 ]]; then [[ $DEBUG -eq 0 ]] && echo "End : $(date -u)" >>download.log [[ $DEBUG -eq 1 ]] && echo "End : $(date -u) *** Simulation ***" >>download.log - [[ $SKIP_PURGE -eq 1 ]] && [[ $DownloadFailed -ne 1 ]] && rm "$LockFilePath" + [[ $DEBUG -eq 0 ]] && [[ $SKIP_PURGE -eq 1 ]] && [[ $DownloadFailed -ne 1 ]] && rm "$LockFilePath" + [[ $DEBUG -eq 0 ]] && [[ $VERIFY_LIBRARY -eq 1 ]] && [[ $RequiresDownload -eq 1 ]] && rm "$LockFilePath" continue fi @@ -687,36 +872,36 @@ if [ $AnyDownloads -eq 1 ]; then ######################################## - echo -e "\033[0;34m Old : $OldZIM\033[0m" + echo -e "${BLUE_REGULAR} Old : $OldZIM${CLEAR}" echo "Old : $OldZIM" >>download.log - echo -e "\033[1;34m New : $NewZIM\033[0m" + echo -e "${BLUE_BOLD} New : $NewZIM${CLEAR}" echo "New : $NewZIM" >>download.log # Check for the new ZIM on disk. if [[ -f "$NewZIMPath" ]]; then # New ZIM found if [[ $DEBUG -eq 0 ]]; then if [[ "$OldZIMPath" == "$NewZIMPath" ]]; then - echo -e "\033[1;32m ✓ Status : New ZIM downloaded succesfully.\033[0m" - echo "✓ Status : New ZIM downloaded succesfully." >>download.log + echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded successfully.${CLEAR}" + echo "✓ Status : New ZIM downloaded successfully." >>download.log # rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM else - echo -e "\033[1;32m ✓ Status : New ZIM downloaded succesfully. Old ZIM purged.\033[0m" - echo "✓ Status : New ZIM downloaded succesfully. Old ZIM purged." >>download.log + echo -e "${GREEN_BOLD} ✓ Status : New ZIM downloaded successfully. Old ZIM purged.${CLEAR}" + echo "✓ Status : New ZIM downloaded successfully. Old ZIM purged." >>download.log [[ -f "$OldZIMPath" ]] && rm "$OldZIMPath" && rm "$OldZIMPath.sha256" 2>/dev/null # Purge old ZIM fi else - echo -e "\033[1;32m ✓ Status : *** Simulated ***\033[0m" + echo -e "${GREEN_BOLD} ✓ Status : *** Simulated ***${CLEAR}" echo "✓ Status : *** Simulated ***" >>download.log fi else # New ZIM not found. Something went wrong, so we will skip this purge. if [[ $DEBUG -eq 0 ]]; then - echo -e "\033[1;31m ✗ Status : New ZIM failed verification. Old ZIM not purged.\033[0m" + echo -e "${RED_BOLD} ✗ Status : New ZIM failed verification. Old ZIM not purged.${CLEAR}" echo "✗ Status : New ZIM failed verification. Old ZIM not purged." >>download.log else if [[ $RequiresDownload -eq 1 ]]; then - echo -e "\033[1;32m ✓ Status : *** Simulated *** New zim exists, old zim purged\033[0m" + echo -e "${GREEN_BOLD} ✓ Status : *** Simulated *** New zim exists, old zim purged${CLEAR}" echo "✓ Status : *** Simulated *** New zim exists, old zim purged" >>download.log else - echo -e "\033[1;33m ✗ Status : *** Simulated *** Zim was skipped, and will not be purged\033[0m" + echo -e "${YELLOW_BOLD} ✗ Status : *** Simulated *** Zim was skipped, and will not be purged${CLEAR}" echo "✗ Status : *** Simulated *** Zim not purged" >>download.log fi fi @@ -735,21 +920,21 @@ if [ $AnyDownloads -eq 1 ]; then unset IFS if [[ ${#HangingFileLocks[@]} -gt 0 ]]; then - echo -e "\033[1;33m5. Cleaning up...\033[0m" + echo -e "${YELLOW_BOLD}5. Cleaning up...${CLEAR}" echo "5. Cleaning up..." >> download.log echo for ((i = 0; i < ${#HangingFileLocks[@]}; i++)); do baseFileName=$(echo "${HangingFileLocks[$i]}" | grep -ioP "(?<=\.~lock\.).*$") - echo -e " \033[1;31mFound broken lock: ${HangingFileLocks[$i]}\033[0m" + echo -e " ${RED_BOLD}Found broken lock: ${HangingFileLocks[$i]}${CLEAR}" echo "Found broken lock: ${HangingFileLocks[$i]}" >> download.log if [[ -f "$ZIMPath$baseFileName" ]]; then - echo -e " \033[1;34mFound abandoned download: $ZIMPath$baseFileName\033[0m" + echo -e " ${BLUE_BOLD}Found abandoned download: $ZIMPath$baseFileName${CLEAR}" echo -e "Found abandoned download: $ZIMPath$baseFileName" >> download.log rm "$ZIMPath$baseFileName" fi if [[ -f "$ZIMPath$baseFileName.sha256" ]]; then - echo -e " \033[1;34mFound abandoned checksum: $ZIMPath$baseFileName.sha256\033[0m" + echo -e " ${BLUE_BOLD}Found abandoned checksum: $ZIMPath$baseFileName.sha256${CLEAR}" echo -e "Found abandoned checksum: $ZIMPath$baseFileName.sha256" >> download.log rm "$ZIMPath$baseFileName.sha256" fi @@ -759,7 +944,12 @@ if [ $AnyDownloads -eq 1 ]; then fi fi else - echo -e "\033[0;32m ✓ Download: Nothing to download.\033[0m" + echo -e "${GREEN_REGULAR} ✓ Download: Nothing to download.${CLEAR}" echo "✓ Download: Nothing to download." >> download.log echo fi + +if [[ "$CALLEDSCRIPTFILE" == "kiwix-zim.sh" ]]; then + echo "WARNING: kiwix-zim has been deprecated in favor of kiwix-zim-updater. kiwix-zim will be a hard link to the new script name until at least 2025-01-01.\n This exit code is nonzero, but the script has not errored out. This is simply so you may notice and switch to using kiwix-zim-updater " >> download.log + exit 2 +fi \ No newline at end of file